Dependency integration¶
Many production files reference other production objects: a USD shot layer references a character layer, a Nuke script reads plates, a timeline references clips, a material references textures. PostProject records those references as dependency relationships. It never parses a file to find them: the host, or an adapter that understands the format, extracts the references and records them.
Record a complete dependency set¶
Dependencies are an observation of one representation’s content, so a host records the complete set each time it extracts references, not individual edges. Each dependency has an open-world kind, a target — an asset whose current representation is used, or one pinned representation — a required or optional flag, and the authored reference exactly as it appears in the file.
After a new fingerprint observation of the source, its recorded set is marked as needing extraction: the old set is kept, but no longer presented as current knowledge until the host records a fresh one.
static pp_error_code_t read_dependency_set(
const pp_production_t *production, const pp_representation_id_t *representation_id,
pp_dependency_set_status_t *out_status, pp_error_t **error) {
pp_dependency_set_t *set = NULL;
uint8_t present = 0;
pp_representation_id_t source_id;
uint64_t recorded_at = 0;
uint64_t count = 0;
pp_error_code_t status =
pp_production_dependency_set(production, *representation_id, &set, error);
if (status == PP_OK) {
status = pp_dependency_set_get(set, &present, &source_id, &recorded_at,
out_status, &count, error);
}
/* Absent knowledge differs from a recorded empty set. */
if (status == PP_OK && !present) {
printf("no dependency set recorded\n");
}
for (uint64_t i = 0; status == PP_OK && present && i < count; ++i) {
pp_dependency_t dependency;
status = pp_dependency_set_get_dependency(set, i, &dependency, error);
if (status == PP_OK) {
printf("%s -> %s (%s)\n", dependency.kind,
dependency.authored_reference != NULL
? dependency.authored_reference
: "-",
dependency.required ? "required" : "optional");
}
}
if (status == PP_OK && present) {
printf("status %u, recorded at revision %llu\n", *out_status,
(unsigned long long)recorded_at);
}
pp_dependency_set_release(set);
return status;
}
static pp_error_code_t record_dependencies(pp_production_t *production,
const pp_representation_id_t *source_id,
const pp_asset_id_t *original_asset,
const pp_representation_id_t *original_id,
pp_error_t **error) {
pp_dependency_t dependencies[2];
memset(dependencies, 0, sizeof dependencies);
dependencies[0].kind = "org.example:source-media";
pp_error_code_t reference_status = pp_object_ref_from_asset(*original_asset, &dependencies[0].target, error);
if (reference_status != PP_OK) return reference_status;
dependencies[0].has_resolved_representation = 1;
dependencies[0].resolved_representation_id = *original_id;
dependencies[0].required = 1;
dependencies[0].authored_reference = "rushes/A001.mov";
dependencies[1].kind = "org.example:timecode-reference";
reference_status = pp_object_ref_from_representation(*original_id, &dependencies[1].target, error);
if (reference_status != PP_OK) return reference_status;
dependencies[1].required = 0;
dependencies[1].authored_reference = "A001.mov#timecode";
pp_read_session_t *view = NULL;
pp_transaction_t *transaction = NULL;
pp_error_code_t status =
pp_production_read_session(production, &view, error);
if (status == PP_OK) {
status = pp_read_session_begin_edit(view, NULL, &transaction, error);
}
pp_read_session_release(view);
if (status == PP_OK) {
/* The call replaces the complete, ordered dependency observation. */
status = pp_transaction_record_dependency_set(transaction, *source_id,
dependencies, 2, error);
}
if (status == PP_OK) {
pp_commit_receipt_t commit_receipt;
status = pp_transaction_commit(transaction, &commit_receipt, error);
}
pp_transaction_release(transaction);
return status;
}
static pp_error_code_t observe_source(pp_production_t *production,
const pp_representation_id_t *source_id,
const uint8_t value[8],
pp_error_t **error) {
pp_transaction_t *transaction = NULL;
pp_read_session_t *view = NULL;
pp_error_code_t status =
pp_production_read_session(production, &view, error);
if (status == PP_OK) {
status = pp_read_session_begin_edit(view, NULL, &transaction, error);
}
pp_read_session_release(view);
view = NULL;
if (status == PP_OK) {
/* A new source fingerprint marks its dependency set for re-extraction. */
status = pp_transaction_record_representation_fingerprint(
transaction, *source_id, HOST_REPRESENTATION_ALGORITHM, 1, value, 8,
error);
}
if (status == PP_OK) {
pp_commit_receipt_t commit_receipt;
status = pp_transaction_commit(transaction, &commit_receipt, error);
}
pp_transaction_release(transaction);
return status;
}
static pp_error_code_t
count_dependencies_by_page(const pp_production_t *production,
const pp_representation_id_t *source_id, uint64_t *out_count,
pp_error_t **error) {
char cursor_storage[2049] = {0};
const char *cursor = NULL;
pp_error_code_t status = PP_OK;
*out_count = 0;
while (status == PP_OK) {
pp_dependency_query_set_t *page = NULL;
status = pp_production_dependencies(production, *source_id, UINT32_C(4),
UINT32_C(1000), UINT32_C(1), cursor,
&page, error);
*out_count += status == PP_OK ? pp_dependency_query_set_count(page) : 0;
const char *next =
status == PP_OK ? pp_dependency_query_set_next_cursor(page) : NULL;
const int copied =
next != NULL
? snprintf(cursor_storage, sizeof cursor_storage, "%s", next)
: 0;
pp_dependency_query_set_release(page);
if (status != PP_OK || copied == 0) {
break;
}
if (copied < 0 || (size_t)copied >= sizeof cursor_storage) {
status = PP_ERROR_INTERNAL;
}
cursor = cursor_storage;
}
return status;
}
std::vector<postproject::DependencyMatch> record_scene_dependencies(
postproject::Production &production, const postproject::RepresentationId &scene_id,
const std::vector<postproject::Dependency> &observed) {
// A dependency set replaces the complete previous observation.
auto transaction = production.readSession().value().edit().value();
transaction.recordDependencySet(scene_id, observed).value();
transaction.commit().value();
if (const auto set = production.dependencySet(scene_id).value()) {
std::cout << set->dependencies.size() << " dependencies, "
<< (set->status == postproject::DependencySetStatus::current
? "current"
: "needs extraction")
<< '\n';
}
std::vector<postproject::DependencyMatch> matches;
std::optional<std::string> cursor;
do {
const auto page = production.dependencies(scene_id, 4, 1000, 1, cursor).value();
matches.insert(matches.end(), page.items.begin(), page.items.end());
cursor = page.next_cursor;
} while (cursor.has_value());
return matches;
}
def record_edit_dependencies(
production: Production,
edit_id: RepresentationId,
camera_asset_id: AssetId,
original_id: RepresentationId,
proxy_id: RepresentationId,
) -> DependencySet | None:
# Always record the complete set: it replaces the previous observation.
dependencies = (
Dependency(
"org.example:clip-reference",
AssetRef(camera_asset_id),
"rushes/A001.mov",
resolved_representation_id=original_id,
),
Dependency(
"org.example:offline-proxy",
RepresentationRef(proxy_id),
"proxies/A001_proxy.mov",
required=False,
),
)
with production.read_session() as view, view.edit() as transaction:
transaction.record_dependency_set(edit_id, dependencies)
transaction.commit()
recorded = production.dependency_set(edit_id) # None: never extracted
if recorded is not None:
print(f"{len(recorded.dependencies)} dependencies, {recorded.status.name}")
return recorded
def all_dependencies(
production: Production, edit_id: RepresentationId
) -> list[DependencyMatch]:
matches: list[DependencyMatch] = []
cursor = None
while True:
page = production.dependencies(
edit_id, max_depth=1, max_representations=1000, limit=1, cursor=cursor
)
matches.extend(page.items)
cursor = page.next_cursor
if cursor is None:
return matches
fn record_comp_dependencies(
production: &mut SqliteProduction,
comp_id: RepresentationId,
plate_asset_id: AssetId,
plate_original_id: RepresentationId,
proxy_id: RepresentationId,
) -> Result<()> {
let dependencies = [
// A floating asset reference, recorded with what it resolved to.
Dependency::new(
None,
DependencyKind::new("org.example:plate")?,
DependencyTarget::Asset(plate_asset_id),
Some(plate_original_id),
true,
"rushes/A001.mov",
)?,
// A reference pinned to one representation.
Dependency::new(
None,
DependencyKind::new("org.example:preview")?,
DependencyTarget::Representation(proxy_id),
None,
false,
"proxies/A001_proxy.mov",
)?,
];
{
let base = production.read_session()?.decision_base();
let mut transaction = production.begin_edit(base)?;
// The set replaces the previous observation as a whole.
transaction.record_dependency_set(comp_id, &dependencies)?;
transaction.commit()?;
}
let set = production
.dependency_set(comp_id)?
.expect("a dependency set was recorded");
println!(
"{} dependencies recorded at revision {}: {:?}",
set.dependencies().len(),
set.recorded_at_revision(),
set.status()
);
for dependency in set.dependencies() {
println!(
"{} -> {:?} ({})",
dependency.kind().as_str(),
dependency.target(),
dependency.authored_reference()
);
}
let limits = DependencyQueryLimits::new(4, 1_000)?;
let mut cursor = None;
loop {
let page = production.dependencies(comp_id, limits, &QueryPageRequest::new(1, cursor)?)?;
for item in page.items() {
println!("depends on {:?} at depth {}", item.target(), item.depth());
}
cursor = page.next_cursor().cloned();
if cursor.is_none() {
return Ok(());
}
}
}
fn dependency_status(
production: &SqliteProduction,
comp_id: RepresentationId,
) -> Result<Option<DependencySetStatus>> {
// After a new fingerprint observation of the comp, the recorded set is
// reported as needing extraction until it is recorded again.
Ok(production.dependency_set(comp_id)?.map(|set| set.status()))
}
# The recorded array replaces the representation's complete dependency set.
cat > dependencies.json <<EOF
[
{
"kind": "org.example:plate",
"target": {"kind": "asset", "id": "$ASSET_ID"},
"resolved_representation_id": "$ORIGINAL_ID",
"authored_reference": "rushes/A001.mov"
},
{
"kind": "org.example:preview",
"target": {"kind": "representation", "id": "$PROXY_ID"},
"required": false,
"authored_reference": "proxies/A001_proxy.mov"
}
]
EOF
BASE=$(postproject --json inspect provenance.pproj | jq -r .decision_base)
RECORDED=$(postproject --json --decision-base "$BASE" dependency record provenance.pproj "$COMP_ID" dependencies.json)
# This is the accepted submission; dependency show reads current stored facts.
test "$(jq -r .changed <<<"$RECORDED")" = true
test "$(jq '.dependencies | length' <<<"$RECORDED")" = 2
test "$(jq '.commit_receipt.revision.sequence > 0' <<<"$RECORDED")" = true
postproject dependency show provenance.pproj "$COMP_ID"
CURSOR=
while :; do
ARGS=(--json dependency dependencies provenance.pproj "$COMP_ID" --limit 1)
[[ -n "$CURSOR" ]] && ARGS+=(--cursor "$CURSOR")
PAGE=$(postproject "${ARGS[@]}")
jq -r '.items[] | "\(.target.kind) \(.target.id) depth \(.depth)"' <<<"$PAGE"
CURSOR=$(jq -r '.next_cursor // empty' <<<"$PAGE")
[[ -z "$CURSOR" ]] && break
done
# Once the comp itself changes, its recorded dependencies need re-extraction.
printf 'comp script v2' > comp/shot010.nk
COMP_RESOURCE_ID=$(postproject --json representation resources provenance.pproj \
"$COMP_ID" | jq -r '.items[0].id')
DECISION_BASE=$(postproject --json inspect provenance.pproj | jq -r .decision_base)
postproject --decision-base "$DECISION_BASE" media fingerprint provenance.pproj "$COMP_RESOURCE_ID" \
comp/shot010.nk
postproject --json dependency show provenance.pproj "$COMP_ID" | jq -r .status
Recording an identical set is a no-op. An explicitly recorded empty set means “inspected, no references”, which is different from never having recorded a set.
The runnable scene/texture fixture
extracts the references actually present in a small JSON scene. The texture
declares an empty reference array, so its empty observation is justified. A
render records the scene as a provenance input; changing the referenced
texture makes that render stale without inventing a dependency on the render.
Query dependencies and dependents¶
Forward queries follow a representation’s dependencies; reverse queries answer “which shots use this character?”. Both are bounded by depth and by the number of representations visited, paged with an opaque cursor, and report separately when a bound truncated the traversal:
static pp_error_code_t
record_and_query_dependencies(pp_production_t *production,
const pp_representation_id_t *source_id,
const pp_asset_id_t *target_asset_id,
const pp_representation_id_t *resolved_id,
pp_error_t **error) {
pp_dependency_t dependency = {0};
dependency.kind = "org.example:clip-reference";
pp_error_code_t reference_status = pp_object_ref_from_asset(*target_asset_id, &dependency.target, error);
if (reference_status != PP_OK) return reference_status;
dependency.has_resolved_representation = UINT8_C(1);
dependency.resolved_representation_id = *resolved_id;
dependency.required = UINT8_C(1);
dependency.authored_reference = "rushes/A001.mov";
pp_read_session_t *view = NULL;
pp_transaction_t *transaction = NULL;
pp_dependency_query_set_t *dependencies = NULL;
pp_dependency_query_set_t *dependents = NULL;
pp_error_code_t status =
pp_production_read_session(production, &view, error);
if (status == PP_OK) {
status = pp_read_session_begin_edit(view, NULL, &transaction, error);
}
pp_read_session_release(view);
if (status == PP_OK) {
status = pp_transaction_record_dependency_set(
transaction, *source_id, &dependency, UINT64_C(1), error);
}
if (status == PP_OK) {
pp_commit_receipt_t commit_receipt;
status = pp_transaction_commit(transaction, &commit_receipt, error);
}
if (status == PP_OK) {
status = pp_production_dependencies(production, *source_id, UINT32_C(4),
UINT32_C(1000), UINT32_C(100), NULL,
&dependencies, error);
}
for (uint64_t i = 0;
status == PP_OK && i < pp_dependency_query_set_count(dependencies);
++i) {
pp_dependency_match_t match;
status = pp_dependency_query_set_get(dependencies, i, &match, error);
if (status == PP_OK) {
printf("dependency at depth %u\n", match.depth);
}
}
pp_object_ref_t target;
pp_error_code_t target_status = pp_object_ref_from_asset(*target_asset_id, &target, error);
if (target_status != PP_OK) return target_status;
if (status == PP_OK) {
status = pp_production_dependents(production, &target, UINT32_C(4),
UINT32_C(1000), UINT32_C(100), NULL,
&dependents, error);
}
if (status == PP_OK &&
(pp_dependency_query_set_traversal_truncated(dependencies) != 0 ||
pp_dependency_query_set_count(dependents) != UINT64_C(1))) {
status = PP_ERROR_INTERNAL;
}
pp_dependency_query_set_release(dependents);
pp_dependency_query_set_release(dependencies);
pp_transaction_release(transaction);
return status;
}
void record_and_query_dependencies(postproject::Production &production,
const postproject::RepresentationId &source_id,
const postproject::AssetId &target_asset_id,
const postproject::RepresentationId &resolved_id) {
const postproject::ObjectRef target = postproject::ObjectRef::asset(target_asset_id);
const postproject::Dependency dependency{
std::nullopt, "org.example:clip-reference", target, resolved_id,
true, "rushes/A001.mov"};
auto transaction = production.readSession().value().edit().value();
transaction.recordDependencySet(source_id, {dependency}).value();
transaction.commit().value();
const auto dependencies = production.dependencies(source_id, 4, 1000, 100).value();
for (const auto &match : dependencies.items) {
std::cout << "dependency at depth " << match.depth << '\n';
}
require(!dependencies.traversal_truncated, "complete dependency traversal");
const auto dependents = production.dependents(target, 4, 1000, 100).value();
require(dependents.items.size() == 1 &&
dependents.items.front().target == postproject::ObjectRef::representation(source_id),
"reverse dependency query");
}
def record_and_query_dependencies(
production: Production,
source_id: RepresentationId,
target_asset_id: AssetId,
resolved_id: RepresentationId,
) -> None:
dependency = Dependency(
kind="org.example:clip-reference",
target=AssetRef(target_asset_id),
authored_reference="rushes/A001.mov",
resolved_representation_id=resolved_id,
)
with production.read_session() as view, view.edit() as transaction:
transaction.record_dependency_set(source_id, (dependency,))
transaction.commit()
dependencies = production.dependencies(
source_id, max_depth=4, max_representations=1000, limit=100
)
for match in dependencies.items:
print(f"dependency {match.target} at depth {match.depth}")
assert not dependencies.traversal_truncated
dependents = production.dependents(
AssetRef(target_asset_id), max_depth=4, max_representations=1000, limit=100
)
assert dependents.items[0].target == RepresentationRef(source_id)
fn record_and_query_dependencies(
production: &mut SqliteProduction,
source_id: RepresentationId,
target_asset_id: AssetId,
resolved_id: RepresentationId,
) -> Result<()> {
let dependency = Dependency::new(
None,
DependencyKind::new("org.example:clip-reference")?,
DependencyTarget::Asset(target_asset_id),
Some(resolved_id),
true,
"rushes/A001.mov",
)?;
{
let base = production.read_session()?.decision_base();
let mut transaction = production.begin_edit(base)?;
transaction.record_dependency_set(source_id, &[dependency])?;
transaction.commit()?;
}
let limits = DependencyQueryLimits::new(4, 1_000)?;
let request = QueryPageRequest::new(100, None)?;
let dependencies = production.dependencies(source_id, limits, &request)?;
for item in dependencies.items() {
println!("dependency {:?} at depth {}", item.target(), item.depth());
}
assert!(!dependencies.traversal_truncated());
let dependents =
production.dependents(DependencyTarget::Asset(target_asset_id), limits, &request)?;
assert_eq!(
dependents.items()[0].target(),
DependencyTarget::Representation(source_id)
);
Ok(())
}
# This small scene actually authors the live reference; rendered frames do not.
printf 'rushes/A001.mov\n' > scene.ppscene
SCENE_ID=$(postproject --json media add production.pproj scene.ppscene | jq -r .representation_id)
cat > dependency.json <<EOF
[
{
"kind": "org.example:media-reference",
"target": {"kind": "asset", "id": "$ASSET_ID"},
"resolved_representation_id": "$ORIGINAL_ID",
"required": true,
"authored_reference": "rushes/A001.mov"
}
]
EOF
BASE=$(postproject --json inspect production.pproj | jq -r .decision_base)
postproject --decision-base "$BASE" dependency record production.pproj "$SCENE_ID" dependency.json
postproject --json dependency dependencies production.pproj "$SCENE_ID" \
--max-depth 4 --max-representations 1000 --limit 100 |
jq '.items[] | {target, depth}'
postproject dependency dependents production.pproj asset "$ASSET_ID" \
--max-depth 4 --max-representations 1000 --limit 100
Cycles are permitted, because composition systems do not guarantee acyclic references. The bounds make every traversal finish regardless. See bounded queries for the paging contract.
Dependencies and staleness¶
A source whose dependency changed is not itself stale: it picks up the change the next time it is opened. Artifacts produced from that source are stale, because the activity captured the dependency closure when it ran. Artifacts and staleness shows the evaluation.