Apply reader-side orientation metadata and stabilize tracker stroke rendering

- Move phone/video rotation metadata handling into readers with an internal clip compatibility mode for legacy projects. FFmpegReader now reports oriented dimensions, applies orientation to decoded frames, and preserves old JSON behavior through reader_orientation_mode.

- Also adjust tracker/object-detection stroke width for preview raster scaling so tracked boxes remain visible during clip scale animations.
This commit is contained in:
Jonathan Thomas
2026-05-08 20:58:53 -05:00
parent eb2411e973
commit 0081cda5ee
13 changed files with 267 additions and 15 deletions
+30
View File
@@ -82,6 +82,7 @@ void Clip::init_settings()
composite = COMPOSITE_SOURCE_OVER;
waveform = false;
waveform_mode = AUDIO_VISUALIZATION_FILLED_WAVEFORM;
reader_orientation_mode = ReaderOrientationMode::Reader;
previous_properties = "";
parentObjectId = "";
@@ -152,6 +153,11 @@ void Clip::init_reader_rotation() {
if (rotation.GetCount() > 0 || !reader)
return;
if (reader->ApplyOrientationMetadata()) {
rotation = Keyframe(0.0f);
return;
}
const auto rotate_meta = reader->info.metadata.find("rotate");
if (rotate_meta == reader->info.metadata.end()) {
// Ensure rotation keyframes always start with a default 0° point.
@@ -359,6 +365,7 @@ void Clip::Reader(ReaderBase* new_reader)
// set parent
if (reader) {
reader->ApplyOrientationMetadata(reader_orientation_mode == ReaderOrientationMode::Reader);
reader->ParentClip(this);
// Init reader info struct
@@ -962,6 +969,15 @@ Json::Value Clip::JsonValue() const {
root["composite"] = composite;
root["waveform"] = waveform;
root["waveform_mode"] = waveform_mode;
switch (reader_orientation_mode) {
case ReaderOrientationMode::LegacyClipTransform:
root["reader_orientation_mode"] = "legacy_clip_transform";
break;
case ReaderOrientationMode::Reader:
default:
root["reader_orientation_mode"] = "reader";
break;
}
root["scale_x"] = scale_x.JsonValue();
root["scale_y"] = scale_y.JsonValue();
root["location_x"] = location_x.JsonValue();
@@ -1034,6 +1050,19 @@ void Clip::SetJsonValue(const Json::Value root) {
// Set parent data
ClipBase::SetJsonValue(root);
// Older project files predate reader-applied orientation metadata and stored
// phone/camera rotation as ordinary clip rotation/scale keyframes.
if (root["reader_orientation_mode"].isNull()) {
reader_orientation_mode = ReaderOrientationMode::LegacyClipTransform;
} else {
const std::string mode = root["reader_orientation_mode"].asString();
if (mode == "legacy_clip_transform") {
reader_orientation_mode = ReaderOrientationMode::LegacyClipTransform;
} else {
reader_orientation_mode = ReaderOrientationMode::Reader;
}
}
// Set data from Json (if key is found)
if (!root["parentObjectId"].isNull()){
parentObjectId = root["parentObjectId"].asString();
@@ -1213,6 +1242,7 @@ void Clip::SetJsonValue(const Json::Value root) {
// mark as managed reader and set parent
if (reader) {
reader->ApplyOrientationMetadata(reader_orientation_mode == ReaderOrientationMode::Reader);
reader->ParentClip(this);
allocated_reader = reader;
}
+6
View File
@@ -104,8 +104,14 @@ namespace openshot {
void init_reader_rotation();
private:
enum class ReaderOrientationMode {
Reader,
LegacyClipTransform
};
bool waveform; ///< Should a waveform be used instead of the clip's image
int waveform_mode; ///< Audio visualization mode used when waveform is enabled
ReaderOrientationMode reader_orientation_mode; ///< Internal project compatibility mode
std::list<openshot::EffectBase*> effects; ///< List of clips on this timeline
bool is_open; ///< Is Reader opened
std::string parentObjectId; ///< Id of the bounding box that this clip is attached to
+79 -8
View File
@@ -20,6 +20,8 @@
#include <sstream>
#include <unistd.h>
#include <QTransform>
#include "FFmpegUtilities.h"
#include "effects/CropHelpers.h"
@@ -110,7 +112,7 @@ FFmpegReader::FFmpegReader(const std::string &path, DurationStrategy duration_st
seek_audio_frame_found(0), seek_video_frame_found(0), last_seek_max_frame(-1), seek_stagnant_count(0),
last_frame(0), largest_frame_processed(0), current_video_frame(0), audio_pts(0), video_pts(0),
hold_packet(false), pts_offset_seconds(0.0), audio_pts_seconds(0.0), video_pts_seconds(0.0),
NO_PTS_OFFSET(-99999), enable_seek(true) {
NO_PTS_OFFSET(-99999), source_width(0), source_height(0), source_rotation(0.0), enable_seek(true) {
// Initialize FFMpeg, and register all formats and codecs
AV_REGISTER_ALL
@@ -713,6 +715,7 @@ void FFmpegReader::Open() {
break;
}
}
UpdateOrientedVideoInfo();
// Init previous audio location to zero
previous_packet_location.frame = -1;
@@ -1032,7 +1035,13 @@ void FFmpegReader::UpdateAudioInfo() {
void FFmpegReader::UpdateVideoInfo() {
if (info.vcodec.length() > 0) {
// Skip init - if info struct already populated
// JSON-loaded readers may already have display-oriented info populated,
// but decoding still needs the raw stream dimensions for orientation.
if (info.has_video && pStream && pCodecCtx) {
source_height = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->height;
source_width = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->width;
UpdateOrientedVideoInfo();
}
return;
}
@@ -1044,8 +1053,10 @@ void FFmpegReader::UpdateVideoInfo() {
// Set values of FileInfo struct
info.has_video = true;
info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
info.height = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->height;
info.width = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->width;
source_height = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->height;
source_width = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->width;
info.height = source_height;
info.width = source_width;
info.vcodec = pCodecCtx->codec->name;
info.video_bit_rate = (pFormatCtx->bit_rate / 8);
@@ -1190,6 +1201,55 @@ void FFmpegReader::UpdateVideoInfo() {
QString str_value = tag->value;
info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
}
UpdateOrientedVideoInfo();
}
void FFmpegReader::UpdateOrientedVideoInfo() {
if (!ApplyOrientationMetadata() || !info.has_video)
return;
if (source_width <= 0 || source_height <= 0) {
source_width = info.width;
source_height = info.height;
}
if (source_width <= 0 || source_height <= 0)
return;
source_rotation = 0.0;
const auto rotate_meta = info.metadata.find("rotate");
if (rotate_meta != info.metadata.end())
source_rotation = strtod(rotate_meta->second.c_str(), nullptr);
const double radians = source_rotation * M_PI / 180.0;
const int oriented_width = static_cast<int>(std::round(
std::fabs(source_width * std::cos(radians)) +
std::fabs(source_height * std::sin(radians))));
const int oriented_height = static_cast<int>(std::round(
std::fabs(source_width * std::sin(radians)) +
std::fabs(source_height * std::cos(radians))));
if (oriented_width > 0 && oriented_height > 0) {
info.width = oriented_width;
info.height = oriented_height;
Fraction size(info.width * info.pixel_ratio.num, info.height * info.pixel_ratio.den);
size.Reduce();
info.display_ratio.num = size.num;
info.display_ratio.den = size.den;
}
}
void FFmpegReader::ApplyFrameOrientation(std::shared_ptr<openshot::Frame> frame) {
if (!ApplyOrientationMetadata() || std::fabs(source_rotation) < 0.0001 || !frame || !frame->has_image_data)
return;
auto image = frame->GetImage();
if (!image || image->isNull())
return;
QImage oriented = image->transformed(QTransform().rotate(source_rotation), Qt::SmoothTransformation);
frame->AddImage(std::make_shared<QImage>(oriented));
}
bool FFmpegReader::GetIsDurationKnown() {
@@ -1672,14 +1732,16 @@ bool FFmpegReader::GetAVFrame() {
AVPixelFormat decoded_pix_fmt = (AVPixelFormat)(decoded_frame->format);
if (decoded_pix_fmt == AV_PIX_FMT_NONE)
decoded_pix_fmt = (AVPixelFormat)(pStream->codecpar->format);
if (AV_ALLOCATE_IMAGE(pFrame, decoded_pix_fmt, info.width, info.height) <= 0) {
const int decoded_width = decoded_frame->width > 0 ? decoded_frame->width : info.width;
const int decoded_height = decoded_frame->height > 0 ? decoded_frame->height : info.height;
if (AV_ALLOCATE_IMAGE(pFrame, decoded_pix_fmt, decoded_width, decoded_height) <= 0) {
throw OutOfMemory("Failed to allocate image buffer", path);
}
av_image_copy(pFrame->data, pFrame->linesize, (const uint8_t**)decoded_frame->data, decoded_frame->linesize,
decoded_pix_fmt, info.width, info.height);
decoded_pix_fmt, decoded_width, decoded_height);
pFrame->format = decoded_pix_fmt;
pFrame->width = info.width;
pFrame->height = info.height;
pFrame->width = decoded_width;
pFrame->height = decoded_height;
pFrame->color_range = decoded_frame->color_range;
pFrame->colorspace = decoded_frame->colorspace;
pFrame->color_primaries = decoded_frame->color_primaries;
@@ -1957,6 +2019,14 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) {
}
}
// Max-size calculations above are in display-oriented coordinates. Scaling
// happens before orientation is applied, so swap the decode bounds for
// quarter-turn sources to preserve detail after rotation.
const int quarter_turn = ((static_cast<int>(std::round(source_rotation / 90.0)) * 90) % 360 + 360) % 360;
if (ApplyOrientationMetadata() && (quarter_turn == 90 || quarter_turn == 270)) {
std::swap(max_width, max_height);
}
// Determine if image needs to be scaled (for performance reasons)
int original_height = src_height;
if (max_width != 0 && max_height != 0 && max_width < width && max_height < height) {
@@ -2056,6 +2126,7 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) {
// Add image with alpha channel (this will be converted to premultipled when needed, but is slower)
f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888, buffer);
}
ApplyFrameOrientation(f);
// Update working cache
working_cache.Add(f);
+9
View File
@@ -158,6 +158,9 @@ namespace openshot {
double audio_stream_duration_seconds = 0.0;
double format_duration_seconds = 0.0;
double inferred_duration_seconds = 0.0;
int source_width = 0;
int source_height = 0;
double source_rotation = 0.0;
// Cached conversion contexts and frames for performance
SwsContext *img_convert_ctx = nullptr; ///< Cached video scaler context
@@ -252,6 +255,12 @@ namespace openshot {
/// Update File Info for video streams
void UpdateVideoInfo();
/// Update display-oriented geometry from source dimensions and orientation metadata
void UpdateOrientedVideoInfo();
/// Apply source orientation metadata to a decoded frame image
void ApplyFrameOrientation(std::shared_ptr<openshot::Frame> frame);
public:
/// Final cache object used to hold final frames
CacheMemory final_cache;
+9
View File
@@ -57,6 +57,7 @@ ReaderBase::ReaderBase()
clip = NULL;
max_decode_width = 0;
max_decode_height = 0;
apply_orientation_metadata = true;
}
// Display file information
@@ -265,3 +266,11 @@ int ReaderBase::MaxDecodeHeight() const {
bool ReaderBase::HasMaxDecodeSize() const {
return max_decode_width > 0 && max_decode_height > 0;
}
void ReaderBase::ApplyOrientationMetadata(bool value) {
apply_orientation_metadata = value;
}
bool ReaderBase::ApplyOrientationMetadata() const {
return apply_orientation_metadata;
}
+7
View File
@@ -80,6 +80,7 @@ namespace openshot
openshot::ClipBase* clip; ///< Pointer to the parent clip instance (if any)
int max_decode_width; ///< Optional maximum decoded frame width (0 disables the limit)
int max_decode_height; ///< Optional maximum decoded frame height (0 disables the limit)
bool apply_orientation_metadata; ///< Apply source orientation metadata while reading frames
public:
@@ -107,6 +108,12 @@ namespace openshot
/// Return true when a maximum decoded frame size is active.
bool HasMaxDecodeSize() const;
/// Set whether readers should apply source orientation metadata to returned frames.
void ApplyOrientationMetadata(bool value);
/// Return whether readers apply source orientation metadata to returned frames.
bool ApplyOrientationMetadata() const;
/// Close the reader (and any resources it was consuming)
virtual void Close() = 0;
+43
View File
@@ -11,11 +11,14 @@
//
// SPDX-License-Identifier: LGPL-3.0-or-later
#include <algorithm>
#include <cmath>
#include <fstream>
#include "TrackedObjectBBox.h"
#include "Clip.h"
#include "TimelineBase.h"
#include "trackerdata.pb.h"
#include <google/protobuf/util/time_util.h>
@@ -174,6 +177,46 @@ BBox TrackedObjectBBox::GetBox(int64_t frame_number)
return interpolatedBBox;
}
double TrackedObjectBBox::ScaledStrokeWidth(int64_t frame_number, int image_width, int image_height) const
{
const double base_width = std::max(0.0, static_cast<double>(stroke_width.GetValue(frame_number)));
Clip* parent_clip = static_cast<Clip*>(ParentClip());
if (!parent_clip || image_width <= 0 || image_height <= 0)
return base_width;
int target_width = image_width;
int target_height = image_height;
if (parent_clip->ParentTimeline()) {
TimelineBase* timeline = static_cast<TimelineBase*>(parent_clip->ParentTimeline());
if (timeline->preview_width > 0)
target_width = timeline->preview_width;
if (timeline->preview_height > 0)
target_height = timeline->preview_height;
}
QSize output_size(image_width, image_height);
switch (parent_clip->scale) {
case SCALE_FIT:
output_size.scale(target_width, target_height, Qt::KeepAspectRatio);
break;
case SCALE_STRETCH:
output_size.scale(target_width, target_height, Qt::IgnoreAspectRatio);
break;
case SCALE_CROP:
output_size.scale(target_width, target_height, Qt::KeepAspectRatioByExpanding);
break;
case SCALE_NONE:
default:
break;
}
if (output_size.width() <= 0 || output_size.height() <= 0)
return base_width;
const double raster_scale_x = static_cast<double>(image_width) / output_size.width();
const double raster_scale_y = static_cast<double>(image_height) / output_size.height();
return base_width * std::sqrt(raster_scale_x * raster_scale_y);
}
// Interpolate the bouding-boxes properties
BBox TrackedObjectBBox::InterpolateBoxes(double t1, double t2, BBox left, BBox right, double target)
{
+3
View File
@@ -184,6 +184,9 @@ namespace openshot
return const_cast<TrackedObjectBBox *>(this)->GetBox(frame_number);
}
/// Return stroke width adjusted for source-to-output raster scaling
double ScaledStrokeWidth(int64_t frame_number, int image_width, int image_height) const;
/// Load the bounding-boxes information from the protobuf file
bool LoadBoxData(std::string inputFilePath);
+2 -1
View File
@@ -104,7 +104,8 @@ std::shared_ptr<Frame> ObjectDetection::GetFrame(std::shared_ptr<Frame> frame, i
// Set the pen for the border
QPen pen(QColor(stroke_rgba[0], stroke_rgba[1], stroke_rgba[2], 255 * stroke_alpha));
pen.setWidth(stroke_width);
pen.setWidthF(trackedObject->ScaledStrokeWidth(
frame_number, frame_image->width(), frame_image->height()));
painter.setPen(pen);
// Set the brush for the background
+2 -1
View File
@@ -110,7 +110,8 @@ std::shared_ptr<Frame> Tracker::GetFrame(std::shared_ptr<Frame> frame, int64_t f
stroke_rgba[0], stroke_rgba[1], stroke_rgba[2],
int(255 * stroke_alpha)
));
pen.setWidth(stroke_width);
pen.setWidthF(trackedData->ScaledStrokeWidth(
frame_number, frame_image->width(), frame_image->height()));
painter.setPen(pen);
QBrush brush(QColor(
+27 -5
View File
@@ -360,7 +360,7 @@ TEST_CASE( "Metadata rotation does not override manual scaling", "[libopenshot][
CHECK(clip.scale_y.GetPoint(0).co.Y == Approx(0.5).margin(0.00001));
}
TEST_CASE( "Metadata rotation scales only default clips", "[libopenshot][clip]" )
TEST_CASE( "Metadata rotation defaults to reader orientation for new clips", "[libopenshot][clip]" )
{
DummyReader rotated(Fraction(24, 1), 640, 480, 48000, 2, 5.0f);
rotated.info.metadata["rotate"] = "90";
@@ -369,9 +369,10 @@ TEST_CASE( "Metadata rotation scales only default clips", "[libopenshot][clip]"
auto_clip.Reader(&rotated);
REQUIRE(auto_clip.rotation.GetCount() == 1);
CHECK(auto_clip.rotation.GetPoint(0).co.Y == Approx(90.0).margin(0.00001));
CHECK(auto_clip.scale_x.GetPoint(0).co.Y == Approx(0.75).margin(0.00001));
CHECK(auto_clip.scale_y.GetPoint(0).co.Y == Approx(0.75).margin(0.00001));
CHECK(auto_clip.rotation.GetPoint(0).co.Y == Approx(0.0).margin(0.00001));
CHECK(auto_clip.scale_x.GetPoint(0).co.Y == Approx(1.0).margin(0.00001));
CHECK(auto_clip.scale_y.GetPoint(0).co.Y == Approx(1.0).margin(0.00001));
CHECK(auto_clip.Reader()->ApplyOrientationMetadata() == true);
DummyReader rotated_custom(Fraction(24, 1), 640, 480, 48000, 2, 5.0f);
rotated_custom.info.metadata["rotate"] = "90";
@@ -382,11 +383,32 @@ TEST_CASE( "Metadata rotation scales only default clips", "[libopenshot][clip]"
custom_clip.Reader(&rotated_custom);
REQUIRE(custom_clip.rotation.GetCount() == 1);
CHECK(custom_clip.rotation.GetPoint(0).co.Y == Approx(90.0).margin(0.00001));
CHECK(custom_clip.rotation.GetPoint(0).co.Y == Approx(0.0).margin(0.00001));
CHECK(custom_clip.scale_x.GetPoint(0).co.Y == Approx(0.5).margin(0.00001));
CHECK(custom_clip.scale_y.GetPoint(0).co.Y == Approx(0.5).margin(0.00001));
}
TEST_CASE( "Clip JSON stores internal reader orientation compatibility mode", "[libopenshot][clip][json]" )
{
Clip new_clip;
Json::Value new_json = new_clip.JsonValue();
REQUIRE(new_json["reader_orientation_mode"].asString() == "reader");
Clip legacy_clip;
legacy_clip.SetJson("{\"reader\":{\"type\":\"DummyReader\",\"width\":640,\"height\":480,\"metadata\":{\"rotate\":\"90\"}}}");
Json::Value legacy_json = legacy_clip.JsonValue();
CHECK(legacy_json["reader_orientation_mode"].asString() == "legacy_clip_transform");
REQUIRE(legacy_clip.Reader() != nullptr);
CHECK_FALSE(legacy_clip.Reader()->ApplyOrientationMetadata());
Clip reader_clip;
reader_clip.SetJson("{\"reader_orientation_mode\":\"reader\",\"reader\":{\"type\":\"DummyReader\",\"width\":480,\"height\":640,\"metadata\":{\"rotate\":\"90\"}}}");
Json::Value reader_json = reader_clip.JsonValue();
CHECK(reader_json["reader_orientation_mode"].asString() == "reader");
REQUIRE(reader_clip.Reader() != nullptr);
CHECK(reader_clip.Reader()->ApplyOrientationMetadata());
}
TEST_CASE( "SetJsonValue restores defaults for empty core transform keyframes", "[libopenshot][clip][json]" )
{
Clip clip;
+44
View File
@@ -15,12 +15,16 @@
#include <sstream>
#include <memory>
#include <QColor>
#include <QImage>
#include "KeyFrame.h"
#include "Coordinate.h"
#include "Clip.h"
#include "Exceptions.h"
#include "FFmpegReader.h"
#include "Fraction.h"
#include "Frame.h"
#include "Point.h"
#include "Timeline.h"
@@ -787,4 +791,44 @@ TEST_CASE( "GetBoxValues", "[libopenshot][keyframe]" )
CHECK(boxValues["h"] == 20.0);
CHECK(boxValues["ang"] == 30.0);
}
TEST_CASE( "Tracker stroke width compensates for preview raster scaling", "[libopenshot][keyframe][tracker]" )
{
auto blue_run_at_center = []() {
Timeline timeline(100, 100, Fraction(30, 1), 44100, 2, ChannelLayout::LAYOUT_STEREO);
Clip parent;
parent.ParentTimeline(&timeline);
Tracker tracker;
tracker.ParentClip(&parent);
tracker.trackedData->ParentClip(&parent);
tracker.trackedData->AddBox(1, 0.5f, 0.5f, 0.4f, 0.4f, 0.0f);
tracker.trackedData->stroke_alpha = Keyframe(1.0);
tracker.trackedData->background_alpha = Keyframe(0.0);
auto image = std::make_shared<QImage>(200, 200, QImage::Format_RGBA8888_Premultiplied);
image->fill(QColor(0, 0, 0, 0));
auto frame = std::make_shared<Frame>(1, 200, 200, "#000000", 0, 0);
frame->AddImage(image);
tracker.GetFrame(frame, 1);
auto rendered = frame->GetImage();
int best_run = 0;
int current_run = 0;
for (int x = 40; x <= 160; ++x) {
const QColor pixel = rendered->pixelColor(x, 100);
const bool blue_pixel = pixel.alpha() > 0 && pixel.blue() > 120 && pixel.red() < 80 && pixel.green() < 80;
if (blue_pixel) {
current_run++;
best_run = std::max(best_run, current_run);
} else {
current_run = 0;
}
}
return best_run;
};
CHECK(blue_run_at_center() >= 3);
}
#endif
+6
View File
@@ -59,6 +59,12 @@ TEST_CASE( "derived class", "[libopenshot][readerbase]" )
t1.SetMaxDecodeSize(0, 0);
CHECK_FALSE(t1.HasMaxDecodeSize());
CHECK(t1.ApplyOrientationMetadata());
t1.ApplyOrientationMetadata(false);
CHECK_FALSE(t1.ApplyOrientationMetadata());
t1.ApplyOrientationMetadata(true);
CHECK(t1.ApplyOrientationMetadata());
t1.Close();
t1.Open();