diff --git a/src/Clip.cpp b/src/Clip.cpp index 98f21b45..c6203975 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -82,6 +82,7 @@ void Clip::init_settings() composite = COMPOSITE_SOURCE_OVER; waveform = false; waveform_mode = AUDIO_VISUALIZATION_FILLED_WAVEFORM; + reader_orientation_mode = ReaderOrientationMode::Reader; previous_properties = ""; parentObjectId = ""; @@ -152,6 +153,11 @@ void Clip::init_reader_rotation() { if (rotation.GetCount() > 0 || !reader) return; + if (reader->ApplyOrientationMetadata()) { + rotation = Keyframe(0.0f); + return; + } + const auto rotate_meta = reader->info.metadata.find("rotate"); if (rotate_meta == reader->info.metadata.end()) { // Ensure rotation keyframes always start with a default 0° point. @@ -359,6 +365,7 @@ void Clip::Reader(ReaderBase* new_reader) // set parent if (reader) { + reader->ApplyOrientationMetadata(reader_orientation_mode == ReaderOrientationMode::Reader); reader->ParentClip(this); // Init reader info struct @@ -962,6 +969,15 @@ Json::Value Clip::JsonValue() const { root["composite"] = composite; root["waveform"] = waveform; root["waveform_mode"] = waveform_mode; + switch (reader_orientation_mode) { + case ReaderOrientationMode::LegacyClipTransform: + root["reader_orientation_mode"] = "legacy_clip_transform"; + break; + case ReaderOrientationMode::Reader: + default: + root["reader_orientation_mode"] = "reader"; + break; + } root["scale_x"] = scale_x.JsonValue(); root["scale_y"] = scale_y.JsonValue(); root["location_x"] = location_x.JsonValue(); @@ -1034,6 +1050,19 @@ void Clip::SetJsonValue(const Json::Value root) { // Set parent data ClipBase::SetJsonValue(root); + // Older project files predate reader-applied orientation metadata and stored + // phone/camera rotation as ordinary clip rotation/scale keyframes. + if (root["reader_orientation_mode"].isNull()) { + reader_orientation_mode = ReaderOrientationMode::LegacyClipTransform; + } else { + const std::string mode = root["reader_orientation_mode"].asString(); + if (mode == "legacy_clip_transform") { + reader_orientation_mode = ReaderOrientationMode::LegacyClipTransform; + } else { + reader_orientation_mode = ReaderOrientationMode::Reader; + } + } + // Set data from Json (if key is found) if (!root["parentObjectId"].isNull()){ parentObjectId = root["parentObjectId"].asString(); @@ -1213,6 +1242,7 @@ void Clip::SetJsonValue(const Json::Value root) { // mark as managed reader and set parent if (reader) { + reader->ApplyOrientationMetadata(reader_orientation_mode == ReaderOrientationMode::Reader); reader->ParentClip(this); allocated_reader = reader; } diff --git a/src/Clip.h b/src/Clip.h index c748c813..e13c3d89 100644 --- a/src/Clip.h +++ b/src/Clip.h @@ -104,8 +104,14 @@ namespace openshot { void init_reader_rotation(); private: + enum class ReaderOrientationMode { + Reader, + LegacyClipTransform + }; + bool waveform; ///< Should a waveform be used instead of the clip's image int waveform_mode; ///< Audio visualization mode used when waveform is enabled + ReaderOrientationMode reader_orientation_mode; ///< Internal project compatibility mode std::list effects; ///< List of clips on this timeline bool is_open; ///< Is Reader opened std::string parentObjectId; ///< Id of the bounding box that this clip is attached to diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 12060d23..fc39431f 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -20,6 +20,8 @@ #include #include +#include + #include "FFmpegUtilities.h" #include "effects/CropHelpers.h" @@ -110,7 +112,7 @@ FFmpegReader::FFmpegReader(const std::string &path, DurationStrategy duration_st seek_audio_frame_found(0), seek_video_frame_found(0), last_seek_max_frame(-1), seek_stagnant_count(0), last_frame(0), largest_frame_processed(0), current_video_frame(0), audio_pts(0), video_pts(0), hold_packet(false), pts_offset_seconds(0.0), audio_pts_seconds(0.0), video_pts_seconds(0.0), - NO_PTS_OFFSET(-99999), enable_seek(true) { + NO_PTS_OFFSET(-99999), source_width(0), source_height(0), source_rotation(0.0), enable_seek(true) { // Initialize FFMpeg, and register all formats and codecs AV_REGISTER_ALL @@ -713,6 +715,7 @@ void FFmpegReader::Open() { break; } } + UpdateOrientedVideoInfo(); // Init previous audio location to zero previous_packet_location.frame = -1; @@ -1032,7 +1035,13 @@ void FFmpegReader::UpdateAudioInfo() { void FFmpegReader::UpdateVideoInfo() { if (info.vcodec.length() > 0) { - // Skip init - if info struct already populated + // JSON-loaded readers may already have display-oriented info populated, + // but decoding still needs the raw stream dimensions for orientation. + if (info.has_video && pStream && pCodecCtx) { + source_height = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->height; + source_width = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->width; + UpdateOrientedVideoInfo(); + } return; } @@ -1044,8 +1053,10 @@ void FFmpegReader::UpdateVideoInfo() { // Set values of FileInfo struct info.has_video = true; info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1; - info.height = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->height; - info.width = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->width; + source_height = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->height; + source_width = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->width; + info.height = source_height; + info.width = source_width; info.vcodec = pCodecCtx->codec->name; info.video_bit_rate = (pFormatCtx->bit_rate / 8); @@ -1190,6 +1201,55 @@ void FFmpegReader::UpdateVideoInfo() { QString str_value = tag->value; info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString(); } + + UpdateOrientedVideoInfo(); +} + +void FFmpegReader::UpdateOrientedVideoInfo() { + if (!ApplyOrientationMetadata() || !info.has_video) + return; + + if (source_width <= 0 || source_height <= 0) { + source_width = info.width; + source_height = info.height; + } + if (source_width <= 0 || source_height <= 0) + return; + + source_rotation = 0.0; + const auto rotate_meta = info.metadata.find("rotate"); + if (rotate_meta != info.metadata.end()) + source_rotation = strtod(rotate_meta->second.c_str(), nullptr); + + const double radians = source_rotation * M_PI / 180.0; + const int oriented_width = static_cast(std::round( + std::fabs(source_width * std::cos(radians)) + + std::fabs(source_height * std::sin(radians)))); + const int oriented_height = static_cast(std::round( + std::fabs(source_width * std::sin(radians)) + + std::fabs(source_height * std::cos(radians)))); + + if (oriented_width > 0 && oriented_height > 0) { + info.width = oriented_width; + info.height = oriented_height; + + Fraction size(info.width * info.pixel_ratio.num, info.height * info.pixel_ratio.den); + size.Reduce(); + info.display_ratio.num = size.num; + info.display_ratio.den = size.den; + } +} + +void FFmpegReader::ApplyFrameOrientation(std::shared_ptr frame) { + if (!ApplyOrientationMetadata() || std::fabs(source_rotation) < 0.0001 || !frame || !frame->has_image_data) + return; + + auto image = frame->GetImage(); + if (!image || image->isNull()) + return; + + QImage oriented = image->transformed(QTransform().rotate(source_rotation), Qt::SmoothTransformation); + frame->AddImage(std::make_shared(oriented)); } bool FFmpegReader::GetIsDurationKnown() { @@ -1672,14 +1732,16 @@ bool FFmpegReader::GetAVFrame() { AVPixelFormat decoded_pix_fmt = (AVPixelFormat)(decoded_frame->format); if (decoded_pix_fmt == AV_PIX_FMT_NONE) decoded_pix_fmt = (AVPixelFormat)(pStream->codecpar->format); - if (AV_ALLOCATE_IMAGE(pFrame, decoded_pix_fmt, info.width, info.height) <= 0) { + const int decoded_width = decoded_frame->width > 0 ? decoded_frame->width : info.width; + const int decoded_height = decoded_frame->height > 0 ? decoded_frame->height : info.height; + if (AV_ALLOCATE_IMAGE(pFrame, decoded_pix_fmt, decoded_width, decoded_height) <= 0) { throw OutOfMemory("Failed to allocate image buffer", path); } av_image_copy(pFrame->data, pFrame->linesize, (const uint8_t**)decoded_frame->data, decoded_frame->linesize, - decoded_pix_fmt, info.width, info.height); + decoded_pix_fmt, decoded_width, decoded_height); pFrame->format = decoded_pix_fmt; - pFrame->width = info.width; - pFrame->height = info.height; + pFrame->width = decoded_width; + pFrame->height = decoded_height; pFrame->color_range = decoded_frame->color_range; pFrame->colorspace = decoded_frame->colorspace; pFrame->color_primaries = decoded_frame->color_primaries; @@ -1957,6 +2019,14 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { } } + // Max-size calculations above are in display-oriented coordinates. Scaling + // happens before orientation is applied, so swap the decode bounds for + // quarter-turn sources to preserve detail after rotation. + const int quarter_turn = ((static_cast(std::round(source_rotation / 90.0)) * 90) % 360 + 360) % 360; + if (ApplyOrientationMetadata() && (quarter_turn == 90 || quarter_turn == 270)) { + std::swap(max_width, max_height); + } + // Determine if image needs to be scaled (for performance reasons) int original_height = src_height; if (max_width != 0 && max_height != 0 && max_width < width && max_height < height) { @@ -2056,6 +2126,7 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { // Add image with alpha channel (this will be converted to premultipled when needed, but is slower) f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888, buffer); } + ApplyFrameOrientation(f); // Update working cache working_cache.Add(f); diff --git a/src/FFmpegReader.h b/src/FFmpegReader.h index 6417bb1d..7a8143e1 100644 --- a/src/FFmpegReader.h +++ b/src/FFmpegReader.h @@ -158,6 +158,9 @@ namespace openshot { double audio_stream_duration_seconds = 0.0; double format_duration_seconds = 0.0; double inferred_duration_seconds = 0.0; + int source_width = 0; + int source_height = 0; + double source_rotation = 0.0; // Cached conversion contexts and frames for performance SwsContext *img_convert_ctx = nullptr; ///< Cached video scaler context @@ -252,6 +255,12 @@ namespace openshot { /// Update File Info for video streams void UpdateVideoInfo(); + /// Update display-oriented geometry from source dimensions and orientation metadata + void UpdateOrientedVideoInfo(); + + /// Apply source orientation metadata to a decoded frame image + void ApplyFrameOrientation(std::shared_ptr frame); + public: /// Final cache object used to hold final frames CacheMemory final_cache; diff --git a/src/ReaderBase.cpp b/src/ReaderBase.cpp index 784c0cd9..ef03090c 100644 --- a/src/ReaderBase.cpp +++ b/src/ReaderBase.cpp @@ -57,6 +57,7 @@ ReaderBase::ReaderBase() clip = NULL; max_decode_width = 0; max_decode_height = 0; + apply_orientation_metadata = true; } // Display file information @@ -265,3 +266,11 @@ int ReaderBase::MaxDecodeHeight() const { bool ReaderBase::HasMaxDecodeSize() const { return max_decode_width > 0 && max_decode_height > 0; } + +void ReaderBase::ApplyOrientationMetadata(bool value) { + apply_orientation_metadata = value; +} + +bool ReaderBase::ApplyOrientationMetadata() const { + return apply_orientation_metadata; +} diff --git a/src/ReaderBase.h b/src/ReaderBase.h index 0b4cf0b4..304ae4cc 100644 --- a/src/ReaderBase.h +++ b/src/ReaderBase.h @@ -80,6 +80,7 @@ namespace openshot openshot::ClipBase* clip; ///< Pointer to the parent clip instance (if any) int max_decode_width; ///< Optional maximum decoded frame width (0 disables the limit) int max_decode_height; ///< Optional maximum decoded frame height (0 disables the limit) + bool apply_orientation_metadata; ///< Apply source orientation metadata while reading frames public: @@ -107,6 +108,12 @@ namespace openshot /// Return true when a maximum decoded frame size is active. bool HasMaxDecodeSize() const; + /// Set whether readers should apply source orientation metadata to returned frames. + void ApplyOrientationMetadata(bool value); + + /// Return whether readers apply source orientation metadata to returned frames. + bool ApplyOrientationMetadata() const; + /// Close the reader (and any resources it was consuming) virtual void Close() = 0; diff --git a/src/TrackedObjectBBox.cpp b/src/TrackedObjectBBox.cpp index b520b054..ddcabcfc 100644 --- a/src/TrackedObjectBBox.cpp +++ b/src/TrackedObjectBBox.cpp @@ -11,11 +11,14 @@ // // SPDX-License-Identifier: LGPL-3.0-or-later +#include +#include #include #include "TrackedObjectBBox.h" #include "Clip.h" +#include "TimelineBase.h" #include "trackerdata.pb.h" #include @@ -174,6 +177,46 @@ BBox TrackedObjectBBox::GetBox(int64_t frame_number) return interpolatedBBox; } +double TrackedObjectBBox::ScaledStrokeWidth(int64_t frame_number, int image_width, int image_height) const +{ + const double base_width = std::max(0.0, static_cast(stroke_width.GetValue(frame_number))); + Clip* parent_clip = static_cast(ParentClip()); + if (!parent_clip || image_width <= 0 || image_height <= 0) + return base_width; + + int target_width = image_width; + int target_height = image_height; + if (parent_clip->ParentTimeline()) { + TimelineBase* timeline = static_cast(parent_clip->ParentTimeline()); + if (timeline->preview_width > 0) + target_width = timeline->preview_width; + if (timeline->preview_height > 0) + target_height = timeline->preview_height; + } + + QSize output_size(image_width, image_height); + switch (parent_clip->scale) { + case SCALE_FIT: + output_size.scale(target_width, target_height, Qt::KeepAspectRatio); + break; + case SCALE_STRETCH: + output_size.scale(target_width, target_height, Qt::IgnoreAspectRatio); + break; + case SCALE_CROP: + output_size.scale(target_width, target_height, Qt::KeepAspectRatioByExpanding); + break; + case SCALE_NONE: + default: + break; + } + if (output_size.width() <= 0 || output_size.height() <= 0) + return base_width; + + const double raster_scale_x = static_cast(image_width) / output_size.width(); + const double raster_scale_y = static_cast(image_height) / output_size.height(); + return base_width * std::sqrt(raster_scale_x * raster_scale_y); +} + // Interpolate the bouding-boxes properties BBox TrackedObjectBBox::InterpolateBoxes(double t1, double t2, BBox left, BBox right, double target) { diff --git a/src/TrackedObjectBBox.h b/src/TrackedObjectBBox.h index 73ce4b66..9f4d13dd 100644 --- a/src/TrackedObjectBBox.h +++ b/src/TrackedObjectBBox.h @@ -184,6 +184,9 @@ namespace openshot return const_cast(this)->GetBox(frame_number); } + /// Return stroke width adjusted for source-to-output raster scaling + double ScaledStrokeWidth(int64_t frame_number, int image_width, int image_height) const; + /// Load the bounding-boxes information from the protobuf file bool LoadBoxData(std::string inputFilePath); diff --git a/src/effects/ObjectDetection.cpp b/src/effects/ObjectDetection.cpp index 6e1ae97c..d71da143 100644 --- a/src/effects/ObjectDetection.cpp +++ b/src/effects/ObjectDetection.cpp @@ -104,7 +104,8 @@ std::shared_ptr ObjectDetection::GetFrame(std::shared_ptr frame, i // Set the pen for the border QPen pen(QColor(stroke_rgba[0], stroke_rgba[1], stroke_rgba[2], 255 * stroke_alpha)); - pen.setWidth(stroke_width); + pen.setWidthF(trackedObject->ScaledStrokeWidth( + frame_number, frame_image->width(), frame_image->height())); painter.setPen(pen); // Set the brush for the background diff --git a/src/effects/Tracker.cpp b/src/effects/Tracker.cpp index 2776ab7a..3cface1e 100644 --- a/src/effects/Tracker.cpp +++ b/src/effects/Tracker.cpp @@ -110,7 +110,8 @@ std::shared_ptr Tracker::GetFrame(std::shared_ptr frame, int64_t f stroke_rgba[0], stroke_rgba[1], stroke_rgba[2], int(255 * stroke_alpha) )); - pen.setWidth(stroke_width); + pen.setWidthF(trackedData->ScaledStrokeWidth( + frame_number, frame_image->width(), frame_image->height())); painter.setPen(pen); QBrush brush(QColor( diff --git a/tests/Clip.cpp b/tests/Clip.cpp index 86ad3113..bf0d6b6a 100644 --- a/tests/Clip.cpp +++ b/tests/Clip.cpp @@ -360,7 +360,7 @@ TEST_CASE( "Metadata rotation does not override manual scaling", "[libopenshot][ CHECK(clip.scale_y.GetPoint(0).co.Y == Approx(0.5).margin(0.00001)); } -TEST_CASE( "Metadata rotation scales only default clips", "[libopenshot][clip]" ) +TEST_CASE( "Metadata rotation defaults to reader orientation for new clips", "[libopenshot][clip]" ) { DummyReader rotated(Fraction(24, 1), 640, 480, 48000, 2, 5.0f); rotated.info.metadata["rotate"] = "90"; @@ -369,9 +369,10 @@ TEST_CASE( "Metadata rotation scales only default clips", "[libopenshot][clip]" auto_clip.Reader(&rotated); REQUIRE(auto_clip.rotation.GetCount() == 1); - CHECK(auto_clip.rotation.GetPoint(0).co.Y == Approx(90.0).margin(0.00001)); - CHECK(auto_clip.scale_x.GetPoint(0).co.Y == Approx(0.75).margin(0.00001)); - CHECK(auto_clip.scale_y.GetPoint(0).co.Y == Approx(0.75).margin(0.00001)); + CHECK(auto_clip.rotation.GetPoint(0).co.Y == Approx(0.0).margin(0.00001)); + CHECK(auto_clip.scale_x.GetPoint(0).co.Y == Approx(1.0).margin(0.00001)); + CHECK(auto_clip.scale_y.GetPoint(0).co.Y == Approx(1.0).margin(0.00001)); + CHECK(auto_clip.Reader()->ApplyOrientationMetadata() == true); DummyReader rotated_custom(Fraction(24, 1), 640, 480, 48000, 2, 5.0f); rotated_custom.info.metadata["rotate"] = "90"; @@ -382,11 +383,32 @@ TEST_CASE( "Metadata rotation scales only default clips", "[libopenshot][clip]" custom_clip.Reader(&rotated_custom); REQUIRE(custom_clip.rotation.GetCount() == 1); - CHECK(custom_clip.rotation.GetPoint(0).co.Y == Approx(90.0).margin(0.00001)); + CHECK(custom_clip.rotation.GetPoint(0).co.Y == Approx(0.0).margin(0.00001)); CHECK(custom_clip.scale_x.GetPoint(0).co.Y == Approx(0.5).margin(0.00001)); CHECK(custom_clip.scale_y.GetPoint(0).co.Y == Approx(0.5).margin(0.00001)); } +TEST_CASE( "Clip JSON stores internal reader orientation compatibility mode", "[libopenshot][clip][json]" ) +{ + Clip new_clip; + Json::Value new_json = new_clip.JsonValue(); + REQUIRE(new_json["reader_orientation_mode"].asString() == "reader"); + + Clip legacy_clip; + legacy_clip.SetJson("{\"reader\":{\"type\":\"DummyReader\",\"width\":640,\"height\":480,\"metadata\":{\"rotate\":\"90\"}}}"); + Json::Value legacy_json = legacy_clip.JsonValue(); + CHECK(legacy_json["reader_orientation_mode"].asString() == "legacy_clip_transform"); + REQUIRE(legacy_clip.Reader() != nullptr); + CHECK_FALSE(legacy_clip.Reader()->ApplyOrientationMetadata()); + + Clip reader_clip; + reader_clip.SetJson("{\"reader_orientation_mode\":\"reader\",\"reader\":{\"type\":\"DummyReader\",\"width\":480,\"height\":640,\"metadata\":{\"rotate\":\"90\"}}}"); + Json::Value reader_json = reader_clip.JsonValue(); + CHECK(reader_json["reader_orientation_mode"].asString() == "reader"); + REQUIRE(reader_clip.Reader() != nullptr); + CHECK(reader_clip.Reader()->ApplyOrientationMetadata()); +} + TEST_CASE( "SetJsonValue restores defaults for empty core transform keyframes", "[libopenshot][clip][json]" ) { Clip clip; diff --git a/tests/KeyFrame.cpp b/tests/KeyFrame.cpp index 49a4d2b2..f3353b1e 100644 --- a/tests/KeyFrame.cpp +++ b/tests/KeyFrame.cpp @@ -15,12 +15,16 @@ #include #include +#include +#include + #include "KeyFrame.h" #include "Coordinate.h" #include "Clip.h" #include "Exceptions.h" #include "FFmpegReader.h" #include "Fraction.h" +#include "Frame.h" #include "Point.h" #include "Timeline.h" @@ -787,4 +791,44 @@ TEST_CASE( "GetBoxValues", "[libopenshot][keyframe]" ) CHECK(boxValues["h"] == 20.0); CHECK(boxValues["ang"] == 30.0); } + +TEST_CASE( "Tracker stroke width compensates for preview raster scaling", "[libopenshot][keyframe][tracker]" ) +{ + auto blue_run_at_center = []() { + Timeline timeline(100, 100, Fraction(30, 1), 44100, 2, ChannelLayout::LAYOUT_STEREO); + Clip parent; + parent.ParentTimeline(&timeline); + + Tracker tracker; + tracker.ParentClip(&parent); + tracker.trackedData->ParentClip(&parent); + tracker.trackedData->AddBox(1, 0.5f, 0.5f, 0.4f, 0.4f, 0.0f); + tracker.trackedData->stroke_alpha = Keyframe(1.0); + tracker.trackedData->background_alpha = Keyframe(0.0); + + auto image = std::make_shared(200, 200, QImage::Format_RGBA8888_Premultiplied); + image->fill(QColor(0, 0, 0, 0)); + auto frame = std::make_shared(1, 200, 200, "#000000", 0, 0); + frame->AddImage(image); + + tracker.GetFrame(frame, 1); + auto rendered = frame->GetImage(); + + int best_run = 0; + int current_run = 0; + for (int x = 40; x <= 160; ++x) { + const QColor pixel = rendered->pixelColor(x, 100); + const bool blue_pixel = pixel.alpha() > 0 && pixel.blue() > 120 && pixel.red() < 80 && pixel.green() < 80; + if (blue_pixel) { + current_run++; + best_run = std::max(best_run, current_run); + } else { + current_run = 0; + } + } + return best_run; + }; + + CHECK(blue_run_at_center() >= 3); +} #endif diff --git a/tests/ReaderBase.cpp b/tests/ReaderBase.cpp index cfbf39a3..3b5257dc 100644 --- a/tests/ReaderBase.cpp +++ b/tests/ReaderBase.cpp @@ -59,6 +59,12 @@ TEST_CASE( "derived class", "[libopenshot][readerbase]" ) t1.SetMaxDecodeSize(0, 0); CHECK_FALSE(t1.HasMaxDecodeSize()); + CHECK(t1.ApplyOrientationMetadata()); + + t1.ApplyOrientationMetadata(false); + CHECK_FALSE(t1.ApplyOrientationMetadata()); + t1.ApplyOrientationMetadata(true); + CHECK(t1.ApplyOrientationMetadata()); t1.Close(); t1.Open();