Fingerprints, verification, and inventory¶
Resolution answers where content can be reached. This guide covers the related question of what the content is: recording that a file’s bytes changed, checking stored fingerprints against the files on disk, finding media on storage that the production does not know yet, and recording technical facts about a file.
All of these follow one rule: reading the filesystem never changes the production by itself. A check reports; only an explicit transaction records.
Record a new fingerprint observation¶
A fingerprint records what an object’s content currently is. It can change over time. When a host knows that a file was replaced in place, for example a re-rendered plate or a re-conformed audio file, it observes the resource’s content again:
The observation fingerprints the file, or the image-sequence directory, at the given path.
It recomputes the fingerprint of every representation that uses the resource.
It stages both in the transaction, and commit records them in one revision.
The previous values move to history.
Hashing happens when the observation is staged, not while the production is locked. The host never computes a representation fingerprint itself.
static pp_error_code_t observe_changed_file(pp_production_t *production,
const pp_resource_id_t *resource_id,
const char *path,
pp_error_t **error) {
pp_transaction_t *transaction = NULL;
pp_read_session_t *view = NULL;
pp_revision_set_t *latest = NULL;
pp_revision_event_set_t *events = NULL;
pp_content_verification_t verification = 0;
pp_content_observation_t outcome = 0;
/* Verification only reads: it compares the file with the stored value. */
/* An image-sequence resource also takes the naming of the files at path;
* NULL uses the naming recorded for that directory. */
pp_error_code_t status = pp_production_verify_resource(
production, *resource_id, path, NULL, &verification, error);
if (status == PP_OK && verification != PP_CONTENT_DIFFERS) {
status = PP_ERROR_INTERNAL;
}
if (status == PP_OK) {
status = pp_production_read_session(production, &view, error);
}
if (status == PP_OK) {
status = pp_read_session_begin_edit(view, NULL, &transaction, error);
}
pp_read_session_release(view);
view = NULL;
if (status == PP_OK) {
/* Stages the new resource fingerprint and every representation
* fingerprint recomputed from it; commit records both in one revision. */
status = pp_transaction_observe_resource_content(
transaction, *resource_id, path, NULL, &outcome, error);
}
if (status == PP_OK && outcome != PP_OBSERVATION_CHANGED) {
status = PP_ERROR_INTERNAL;
}
if (status == PP_OK) {
pp_commit_receipt_t commit_receipt;
status = pp_transaction_commit(transaction, &commit_receipt, error);
}
pp_transaction_release(transaction);
transaction = NULL;
/* Observing the same content again changes nothing and records nothing. */
if (status == PP_OK) {
status = pp_production_read_session(production, &view, error);
}
if (status == PP_OK) {
status = pp_read_session_begin_edit(view, NULL, &transaction, error);
}
pp_read_session_release(view);
view = NULL;
if (status == PP_OK) {
status = pp_transaction_observe_resource_content(
transaction, *resource_id, path, NULL, &outcome, error);
}
if (status == PP_OK && outcome != PP_OBSERVATION_UNCHANGED) {
status = PP_ERROR_INTERNAL;
}
if (status == PP_OK) {
pp_commit_receipt_t commit_receipt;
status = pp_transaction_commit(transaction, &commit_receipt, error);
}
if (status == PP_OK) {
status = pp_production_latest_revision(production, &latest, error);
}
if (status == PP_OK) {
pp_revision_id_t revision_id;
pp_transaction_id_t transaction_id;
uint64_t sequence = 0;
int64_t committed_at = 0;
const char *origin_name, *origin_version, *origin_uri, *message;
status = pp_revision_set_get(latest, 0, &revision_id, &sequence,
&transaction_id, &committed_at, &origin_name,
&origin_version, &origin_uri, &message, error);
if (status == PP_OK) {
status = pp_production_revision_events(production, revision_id, &events,
error);
}
}
for (uint64_t i = 0;
status == PP_OK && i < pp_revision_event_set_count(events); ++i) {
pp_revision_event_t event;
status = pp_revision_event_set_get(events, i, &event, error);
if (status == PP_OK) {
printf("event kind %u: %s v%u\n", event.kind, event.fingerprint_algorithm,
event.fingerprint_version);
}
}
pp_revision_event_set_release(events);
pp_revision_set_release(latest);
pp_transaction_release(transaction);
return status;
}
std::vector<postproject::RevisionEvent>
observe_new_content(postproject::Production &production,
const postproject::ResourceId &resource_id,
const std::string &path) {
// Verification only reads: it compares the file with the stored value.
if (production.verifyResource(resource_id, path).value() !=
postproject::ContentVerification::differs) {
throw std::runtime_error("content is unchanged");
}
// Stages the new resource fingerprint and every representation fingerprint
// recomputed from it; commit records both in one revision.
auto transaction = production.readSession().value().edit().value();
if (transaction.observeResourceContent(resource_id, path).value() !=
postproject::ContentObservationOutcome::changed) {
throw std::runtime_error("content is unchanged");
}
transaction.commit().value();
const auto observed = production.latestRevision().value()->id;
// Observing the same content again changes nothing and records nothing.
auto again = production.readSession().value().edit().value();
if (again.observeResourceContent(resource_id, path).value() !=
postproject::ContentObservationOutcome::unchanged) {
throw std::runtime_error("content changed again");
}
again.commit().value();
if (production.latestRevision().value()->id != observed) {
throw std::runtime_error("an unchanged observation was recorded");
}
const auto events = production.revisionEvents(observed).value();
for (const auto &event : events) {
std::visit(
[](const auto &payload) {
using Payload = std::decay_t<decltype(payload)>;
if constexpr (std::is_same_v<
Payload,
postproject::ResourceFingerprintObservedEvent>) {
std::cout << "resource fingerprint observed: " << payload.algorithm
<< '\n';
} else if constexpr (
std::is_same_v<
Payload,
postproject::RepresentationFingerprintObservedEvent>) {
std::cout << "representation fingerprint observed: "
<< payload.algorithm << '\n';
}
},
event.payload);
}
return events;
}
def observe_fingerprints(
production: Production, representation: Representation, path: Path
) -> tuple[RevisionEvent, ...]:
(resource,) = representation.resources
# Verification only reads: it compares the file with the stored value.
assert production.verify_resource(resource.id, path) is ContentVerification.DIFFERS
with production.read_session() as view, view.edit() as transaction:
# Stages the new resource fingerprint and every representation
# fingerprint recomputed from it; commit records both in one revision.
outcome = transaction.observe_resource_content(resource.id, path)
assert outcome is ContentObservationOutcome.CHANGED
transaction.commit()
latest = production.latest_revision
assert latest is not None
# Observing the same content again changes nothing and records nothing.
with production.read_session() as view, view.edit() as transaction:
outcome = transaction.observe_resource_content(resource.id, path)
assert outcome is ContentObservationOutcome.UNCHANGED
transaction.commit()
assert production.latest_revision == latest
events = production.revision_events[latest.id]
for event in events:
print(f"event {event.position}: {type(event.payload).__name__}")
return events
fn observe_changed_original(
production: &mut SqliteProduction,
original_id: RepresentationId,
path: &Path,
) -> Result<()> {
let since = production
.latest_revision()?
.map_or(0, |revision| revision.sequence());
let view = production.read_session()?;
let resource_id = view.read().resources(original_id)?[0].id();
let usage = resource_usage(view.read(), resource_id)?;
// Verification only reads: it compares the file with the stored value.
let (representation, resources) = &usage[0];
let resource = resources
.iter()
.find(|resource| resource.id() == resource_id);
let verification = verify_resource_content(
resource.expect("representation lists the resource"),
representation.content_structure(),
path,
None,
)?;
assert_eq!(verification, ContentVerification::Differs);
// The new resource value and every representation recomputed from it are
// recorded in one revision. Recording an unchanged observation adds none.
// An image-sequence resource also needs the naming of the files at path.
let observation = observe_resource_content(resource_id, &usage, path, None)?;
assert_eq!(observation.outcome(), ContentObservationOutcome::Changed);
{
let mut transaction = production.begin_edit(view.decision_base())?;
transaction.record_resource_fingerprint(resource_id, observation.resource())?;
if let Some(facts) = observation.file_facts() {
transaction.record_resource_file_facts(resource_id, facts)?;
}
for (representation_id, fingerprint) in observation.representations() {
transaction.record_representation_fingerprint(*representation_id, fingerprint)?;
}
transaction.commit()?;
}
for revision in production.changes_since(since, 10)? {
for event in production.events_for_revision(revision.id())? {
println!("revision {}: {:?}", revision.sequence(), event.kind());
}
}
Ok(())
}
printf 'regraded camera original' > moved/A001.mov
# Content changed, and PostProject's own fingerprint says so.
postproject media verify-content media.pproj "$RESOURCE_ID" moved/A001.mov
# Records the new resource fingerprint, and recomputes every representation
# using the resource, in one revision. The outcome is "changed".
DECISION_BASE=$(postproject --json inspect media.pproj | jq -r .decision_base)
postproject --json --decision-base "$DECISION_BASE" media fingerprint media.pproj "$RESOURCE_ID" moved/A001.mov |
jq '{outcome, resource_fingerprint, representation_fingerprints}'
REVISION_ID=$(postproject --json revisions latest media.pproj | jq -r .id)
# Observing the same content again reports "unchanged" and records nothing.
DECISION_BASE=$(postproject --json inspect media.pproj | jq -r .decision_base)
postproject --json --decision-base "$DECISION_BASE" media fingerprint media.pproj "$RESOURCE_ID" moved/A001.mov |
jq -r .outcome
test "$(postproject --json revisions latest media.pproj | jq -r .id)" = "$REVISION_ID"
postproject --json revisions events media.pproj "$REVISION_ID" |
jq -r '.[] | "\(.position) \(.kind)"'
The observation reports its outcome (ADR 0035):
unchanged means the content matches the stored fingerprint;
changed means it differs, and the new content is recorded;
first means no fingerprint in a domain PostProject computes was stored, for example when only a host’s own hash was recorded.
Explicit observations require a read-bound edit, including first and unchanged values (ADR 0056). Unchanged content creates no revision unless file facts change or it completes a pending representation recomputation. A stale observation still checks its conflict key even if the submitted value matches current facts. A host therefore observes without verifying first, whenever it may matter: after every render or copy, or when it opens a project. A job worker that produces or replaces media observes the new content before completing, so artifact evaluation sees current evidence.
A representation fingerprint is derived from its resources’ fingerprints. A resource fingerprint recorded without an observation, for example a value computed elsewhere, marks each owning representation for recomputation. Until its recomputed fingerprint is recorded as well, artifact evaluation treats the affected knowledge as pending rather than current.
Compute and compare content without recording¶
Computing a file’s fingerprint needs no production. It returns exactly the value that import records, which is useful for caching, for handing the value to another process, or for checking a file before importing it:
static pp_error_code_t print_file_fingerprint(const char *path,
pp_error_t **error) {
pp_fingerprint_t *fingerprint = NULL;
pp_error_code_t status = pp_fingerprint_file(path, &fingerprint, error);
const char *algorithm = NULL;
uint16_t version = 0;
const uint8_t *value = NULL;
uint64_t length = 0;
if (status == PP_OK) {
status = pp_fingerprint_get(fingerprint, &algorithm, &version, &value,
&length, error);
}
if (status == PP_OK) {
printf("%s v%u: %llu bytes, first %02x\n", algorithm, version,
(unsigned long long)length, value[0]);
}
pp_fingerprint_release(fingerprint);
return status;
}
void print_file_fingerprint(const std::string &path) {
// The same value import records; computing it records nothing.
const postproject::Fingerprint fingerprint = postproject::fingerprintFile(path).value();
std::cout << fingerprint.algorithm << " v" << fingerprint.version << ": "
<< fingerprint.value.size() << " bytes\n";
}
def print_file_fingerprint(path: Path) -> Fingerprint:
# The same value import records; computing it records nothing.
fingerprint = fingerprint_file(path)
print(f"{fingerprint.algorithm} v{fingerprint.version}: {fingerprint.value.hex()}")
return fingerprint
fn print_file_fingerprint(path: &Path) -> Result<()> {
// The same value import records; computing it records nothing.
let report = fingerprint_file(path)?;
let fingerprint = report.fingerprint();
println!(
"{} v{}: {} bytes",
fingerprint.algorithm(),
fingerprint.version(),
fingerprint.value().len()
);
Ok(())
}
The CLI has no command that only computes a fingerprint. postproject media verify-content compares a file with a stored resource.
Verifying compares the content at a path with a resource’s stored fingerprints, and never records anything. The fingerprint-observation example above verifies before it observes. The result is one of three values:
matches: the content is unchanged;
differs: the content changed;
not comparable: the resource has only fingerprints that PostProject cannot compute (see below), so nothing was compared.
For an image sequence, the path is a directory, and verifying and observing also take the naming of its files there. Without one, the naming recorded for that directory is used. Neither fingerprint contains file names, so a renamed copy verifies as a match (ADR 0038). A sequence fingerprint of version 1, from before schema 15, is no longer computed: observe the sequence once to record a version 2 value that can be compared.
Keep a host’s own content hashes¶
Many hosts already hash their media, for example an editor’s head-and-tail digest, a MAM checksum, or a camera’s clip hash. Record such a value as a resource fingerprint in its own domain, and PostProject keeps it byte for byte:
Name the algorithm after your application. The algorithm identifier is 1–64 ASCII letters, digits,
-, or_, for exampleexample-editor-md5-head-tail. Do not reuse a generic name such asmd5unless the value is exactly that digest of every byte.Increment the version whenever the computation changes. Values from different versions are never compared with each other.
Keep PostProject’s own fingerprint as well. Importing a file records it. PostProject can only verify domains it computes itself; every other domain is foreign to it (ADR 0028).
Observe the content in the same transaction. A representation fingerprint covers every fingerprint of its resources, including yours, so recording a host hash leaves each owning representation pending recomputation. Observing the resource’s content in the same transaction recomputes them. After this step no representation is left pending, and artifact evaluation treats the knowledge as current.
Foreign fingerprints never prevent resolution. When a resource has no fingerprint that PostProject can compute, the resolver works as it does for a resource without fingerprints:
It proposes files with the recorded name, and ranks them by size, parent path, and technical inspection.
It never reports such a candidate as exact.
It attaches
FingerprintNotVerifiedevidence, whose detail lists the unchecked domains as<algorithm> version <n>.
The host should hash each such candidate with its own algorithm and confirm the locator only when the values agree. Resolution never commits a candidate by itself, so this check fits the normal confirmation step.
Verify content during resolution¶
By default resolution trusts known locators whose files exist. Verification additionally recomputes the fingerprints of content found at known locators and reports a mismatch as evidence, so replaced or corrupted files are noticed. Verification reads every byte it checks and is opt-in for that reason:
static pp_error_code_t verify_contents(const pp_production_t *production,
const pp_asset_id_t *asset_id,
uint64_t *out_verified,
pp_error_t **error) {
pp_resolution_options_t *options = NULL;
pp_resolution_set_t *resolutions = NULL;
/* Content mode re-fingerprints files at known locators instead of trusting
* their presence. */
pp_resolution_config_t config = PP_RESOLUTION_CONFIG_INIT;
config.verification = PP_VERIFY_CONTENT;
pp_error_code_t status = pp_resolution_options_create(&config, &options, error);
if (status == PP_OK) {
status = pp_production_resolve_assets(production, asset_id, 1, options,
&resolutions, error);
}
pp_resolution_options_release(options);
const uint64_t count =
status == PP_OK ? pp_resolution_set_representation_count(resolutions) : 0;
for (uint64_t r = 0; status == PP_OK && r < count; ++r) {
pp_asset_id_t resolved_asset_id;
pp_representation_id_t representation_id;
pp_representation_availability_t availability;
uint64_t resource_count = 0, issue_count = 0;
status = pp_resolution_set_get_representation(
resolutions, r, &resolved_asset_id, &representation_id, &availability,
&resource_count, &issue_count, error);
for (uint64_t s = 0; status == PP_OK && s < resource_count; ++s) {
pp_resource_id_t resource_id;
pp_resource_resolution_state_t state;
uint64_t candidate_count = 0, evidence_count = 0;
status = pp_resolution_set_get_resource_state(resolutions, r, s, &state, error);
if (status != PP_OK) break;
status = pp_resolution_set_get_resource(resolutions, r, s, state, &resource_id,
&state, &candidate_count,
&evidence_count, error);
if (status == PP_OK && state == PP_RESOURCE_ONLINE_AT_KNOWN_LOCATOR) {
++*out_verified;
}
if (status == PP_OK && state == PP_RESOURCE_RESOLUTION_ERROR) {
/* PP_EVIDENCE_FINGERPRINT_MISMATCH: the content was replaced. */
printf("resource %llu content differs\n", (unsigned long long)s);
}
}
}
pp_resolution_set_release(resolutions);
return status;
}
std::size_t verify_contents(const postproject::Production &production,
const postproject::AssetId &asset_id) {
// Content mode re-fingerprints files at known locators instead of trusting
// their presence.
postproject::ResolutionConfig config;
config.verification = postproject::VerificationMode::content;
auto options = postproject::ResolutionOptions::create(config).value();
std::size_t verified = 0;
for (const auto &representation :
production.resolveAsset(asset_id, options).value()) {
for (const auto &resource : representation.resources) {
// value() exposes the applicable case without a second stored state.
if (const auto *known = std::get_if<postproject::ResourceOnlineAtKnownLocator>(
&resource.outcome.value())) {
if (known->candidates.size() != 1) return 0;
++verified;
} else if (resource.state() ==
postproject::ResourceResolutionState::error) {
// fingerprint_mismatch evidence: the content was replaced.
std::cout << "content differs for " << resource.evidence.size()
<< " reason(s)\n";
}
}
}
return verified;
}
def verify_contents(production: Production, asset_id: AssetId) -> int:
# Content mode re-fingerprints files at known locators instead of trusting
# their presence.
verified = 0
for representation in production.resolve(
asset_id, verification=VerificationMode.CONTENT
):
for resource in representation.resources:
if isinstance(resource.outcome, ResourceOnlineAtKnownLocator):
assert resource.outcome.candidate.evidence
verified += 1
elif resource.state is ResourceResolutionState.ERROR:
# FINGERPRINT_MISMATCH evidence: the content was replaced.
print(f"{resource.resource_id} content differs")
return verified
fn verify_contents(
production: &SqliteProduction,
asset_id: AssetId,
) -> Result<Vec<(ResourceId, ResourceResolutionState)>> {
// Content mode re-fingerprints files at known locators instead of trusting
// their presence.
let resolver = MediaResolver::new(ResolverOptions {
verification: VerificationMode::Content,
..ResolverOptions::default()
})?;
let mut inputs = Vec::new();
for representation in production.representations(asset_id)? {
for resource in production.resources(representation.id())? {
let locators = production.locators(resource.id())?;
inputs.push((representation.clone(), resource, locators));
}
}
let items = inputs
.iter()
.map(|(representation, resource, locators)| {
ResolutionItem::new(resource, representation.content_structure(), locators)
})
.collect::<Vec<_>>();
let scope = SearchScope::new(production.media_roots()?, Vec::new());
let mut states = Vec::new();
for resolution in resolver.resolve(&items, &scope)? {
for evidence in resolution.evidence() {
println!("{}: {:?}", resolution.resource_id(), evidence.kind());
}
states.push((resolution.resource_id(), resolution.state()));
}
Ok(states)
}
printf 'truncated' > proxies/A001_proxy.mov
# --verify recomputes stored fingerprints for content at known locators, so
# changed bytes are reported instead of trusted.
postproject --json media resolve media.pproj "$ASSET_ID" \
--root-map rushes="$PWD/moved" --verify |
jq -r '.resolutions[] | "\(.representation_id) \(.availability) \([.resources[].evidence[].kind])"'
A verification mismatch is a report. Record the new observation, as shown above, only after the integration has decided the new bytes are correct.
FingerprintMismatch means that PostProject compared the content in one of its
own domains and the content differs. If a resource only has foreign
fingerprints, verification reports the known locator as online with
FingerprintNotVerified evidence, because nothing was compared. The host’s own
check decides that case.
Inventory storage¶
An inventory scan walks the mapped media roots and classifies what it finds: known resources that are online, partial, missing, or changed since they were recorded; new media that no representation references yet; duplicate and ambiguous relink candidates; and roots that are unmapped or unavailable on this machine. It never changes the production. An optional sidecar cache lets repeated scans skip fingerprinting unchanged files; the cache is machine-local and holds no production knowledge.
Inventory scanning is not exposed through the C ABI. Run postproject media inventory from a host tool, or resolve assets with
pp_production_resolve_assets.
Inventory scanning is not exposed through the C++ wrapper. Run postproject media inventory from a host tool, or resolve assets with
Production::resolveAssets.
Inventory scanning is not exposed through the Python binding. Run postproject media inventory with subprocess, or resolve assets with
Production.resolve.
fn scan_rushes(
production: &SqliteProduction,
rushes: &Path,
cache: &Path,
) -> Result<Vec<InventoryCategory>> {
let mappings = [MediaRootMapping::new("rushes", rushes)?];
// The cache is disposable; it only avoids re-hashing unchanged files.
let report = InventoryScanner::default().scan(production, &mappings, Some(cache))?;
for item in report.items() {
println!(
"{:?}: {:?} {:?}",
item.category(),
item.uri(),
item.detail()
);
}
println!(
"{} entries visited, {} fingerprints computed",
report.stats().entries_visited,
report.stats().fingerprints_computed
);
Ok(report.items().iter().map(InventoryItem::category).collect())
}
# Inventory classifies files under mapped roots without changing the
# production; the cache only speeds up repeated scans.
postproject --json media inventory media.pproj --root-map rushes="$PWD/moved" \
--cache inventory.cache > inventory.json
jq -r '.items[] | "\(.category) \(.uri)"' inventory.json
jq '.stats' inventory.json
Record technical inspection¶
The optional inspection adapter runs ffprobe as a subprocess and records a
bounded technical summary of the container and its streams as one structured
metadata assertion in the https://postproject.org/ns/technical-media/1
vocabulary, attached to the representation. A missing ffprobe is reported as a capability gap, not an
error, and PostProject never ships or downloads the executable.
Inspection is not exposed through the C ABI. A host that already inspects media
can record its own findings as typed metadata with
pp_transaction_add_metadata_value.
Inspection is not exposed through the C++ wrapper. A host that already inspects
media can record its own findings as typed metadata
with Transaction::addMetadataValue.
Inspection is not exposed through the Python binding. A host that already
inspects media can record its own findings as typed
metadata with Edit.add_metadata.
fn inspect_original(
production: &mut SqliteProduction,
original_id: RepresentationId,
media: &Path,
ffprobe: &Path,
) -> Result<bool> {
let outcome = FfprobeInspector::with_executable(ffprobe).inspect(media)?;
let metadata = match outcome {
InspectionOutcome::Inspected(metadata) => metadata,
// A missing or failing ffprobe is reported, not raised as an error.
other => {
println!("no technical metadata: {other:?}");
return Ok(false);
}
};
let target = ObjectRef::Representation(original_id);
{
let mut transaction = production.begin_transaction()?;
for assertion in metadata.assertions() {
transaction.add_metadata_value(target, assertion.property(), assertion.value())?;
}
transaction.commit()?;
}
let stored = TechnicalMetadata::from_assertions(&production.metadata(target)?);
Ok(stored == Some(metadata))
}
# A stand-in for ffprobe that prints a fixed probe result.
cat > ffprobe-fake <<'EOF'
#!/bin/sh
cat <<'JSON'
{"format": {"format_name": "mov,mp4", "duration": "2.5"},
"streams": [{"index": 0, "codec_type": "video", "codec_name": "prores",
"width": 1920, "height": 1080, "avg_frame_rate": "24000/1001"}]}
JSON
EOF
chmod +x ffprobe-fake
printf 'camera B' > card/B001.mov
postproject --json media add media.pproj card/B001.mov --name "Camera B" \
--inspect --ffprobe "$PWD/ffprobe-fake" > inspected.json
jq -r '.inspections[] | "\(.path): \(.status)"' inspected.json
postproject --json metadata list media.pproj representation \
"$(jq -r .representation_id inspected.json)" |
jq '.[] | select(.vocabulary == "https://postproject.org/ns/technical-media/1")'
Inspection results are observations of one file at one time. They do not replace the fingerprint as evidence of identity.