ZiB Network

// Submit vision AI job
const { job_id } = await zib.submitVisionAI(objectId, 'standard');

// Poll until complete (typically 1–5 min depending on video length)
let status;
do {
  await new Promise(r => setTimeout(r, 3000));
  status = await zib.getComputeStatus(job_id);
} while (status.status !== 'complete' && status.status !== 'failed');

if (status.status === 'failed') {
  throw new Error(status.error_message);
}

// Fetch the sidecar — standard ZiB CDN URL, no special auth needed
const sidecarUrl = zib.getCdnUrl(status.sidecar_file_id);
const sidecar = await fetch(sidecarUrl).then(r => r.json());
console.log(sidecar.scene_understanding.title_suggestion);

POST /v1/api/compute/{objectId}
Authorization: Bearer ACCESS_KEY:SECRET_KEY
Content-Type: application/json

{
  "vision": "standard"
}

{
  "zib_sidecar_version": "1.0",
  "video_object_id": "zib_abc123",
  "generated_at": "2026-03-17T10:00:00Z",
  "processing_time_ms": 4821,
  "models_used": {
    "transcription": "recommended",
    "vision": "standard"
  },

  "video": {
    "duration_s": 612.4,
    "duration_ms": 612400,
    "width": 1920,
    "height": 1080,
    "aspect_ratio": "16:9",
    "fps": 29.97,
    "codec": "h264",
    "audio_channels": 2,
    "audio_sample_rate": 44100,
    "file_size_bytes": 284720384,
    "bitrate_kbps": 3720
  },

  "transcript": {
    "language": "en",
    "language_confidence": 0.98,
    "full_text": "Welcome back to the channel...",
    "word_count": 1842,
    "speaking_pace_wpm": 148,
    "silence_ratio": 0.12,
    "multiple_speakers": false,
    "segments": [
      {
        "index": 0,
        "start_s": 0.0,   "end_s": 4.2,
        "start_ms": 0,    "end_ms": 4200,
        "text": "Welcome back to the channel.",
        "speaker": "speaker_1",
        "confidence": 0.97,
        "intent": "hook",
        "viewer_engagement_prediction": "high",
        "is_hook_moment": true,
        "is_payoff_moment": false,
        "cta_detected": false,
        "cta_text": null,
        "question_posed": null,
        "key_claim": null
      }
    ]
  },

  "subtitles": {
    "language": "en",
    "srt_object_id": "zib_abc123.en.srt",
    "vtt_object_id": "zib_abc123.en.vtt",
    "srt_inline": "1\n00:00:00,000 --> 00:00:04,200\nWelcome back to the channel.\n\n...",
    "vtt_inline": "WEBVTT\n\n00:00:00.000 --> 00:00:04.200\nWelcome back to the channel.\n\n..."
  },

  "chapters": [
    {
      "index": 0,
      "title": "Introduction",
      "summary": "Host explains privacy limitations of centralised cloud storage.",
      "start_s": 0.0,   "end_s": 142.0,
      "start_ms": 0,    "end_ms": 142000,
      "keywords": ["introduction", "decentralised storage", "privacy"],
      "dominant_intent": "hook",
      "engagement_level": "high"
    }
  ],
  "chapters_youtube_format": "0:00 Introduction\n2:22 How ZiB Works\n6:20 Getting Started",

  "scene_understanding": {
    "executive_summary": "A single presenter in a professional home office...",
    "title_suggestion": "ZiB Network: Private Decentralised Storage Explained",
    "description_suggestion": "Learn how ZiB Network solves the privacy problems of traditional cloud...",
    "content_format": "tutorial",
    "narrative_arc": "problem_solution",
    "target_audience": "Intermediate to advanced software developers.",
    "assumed_knowledge_level": "intermediate",
    "unique_value": "Hands-on technical explanation with concrete failure demonstrations.",
    "series_indicators": "References 'last week's video on encryption' at 2:14.",
    "guest_present": false,
    "monetisation_notes": "No sponsored segments. Strong CTA at 9:45.",

    "production": {
      "quality": "prosumer",
      "setting": "home_office",
      "cut_style": "moderate",
      "talking_head_ratio": 0.78,
      "has_screen_recording": true,
      "has_broll": false,
      "has_slides": false,
      "has_lower_thirds": true,
      "has_captions_burned_in": false,
      "has_graphic_overlays": true,
      "presenter_count": 1,
      "consistent_framing": true,
      "consistent_lighting": true
    },

    "engagement_map": {
      "high_engagement_segments": [
        { "start_s": 0.0, "end_s": 45.0, "start_ms": 0, "end_ms": 45000,
          "reason": "Strong hook, high presenter energy" }
      ],
      "low_engagement_risk_segments": [
        { "start_s": 95.0, "end_s": 142.0, "start_ms": 95000, "end_ms": 142000,
          "reason": "Slow transition, background context" }
      ],
      "hook_moments": [
        { "timestamp_s": 8.4, "timestamp_ms": 8400,
          "reason": "Provocative claim: most cloud storage can be read by the provider" }
      ],
      "payoff_moments": [
        { "timestamp_s": 187.0, "timestamp_ms": 187000,
          "reason": "Live demonstration of node failure recovery" }
      ],
      "cta_moments": [
        { "timestamp_s": 585.0, "timestamp_ms": 585000,
          "text": "Link to GitHub in description", "type": "link" }
      ]
    },

    "on_screen_text": [
      { "timestamp_s": 12.0, "timestamp_ms": 12000,
        "text": "How ZiB Storage Works", "type": "title_card" }
    ],

    "keyframes": [
      {
        "timestamp_s": 0.0, "timestamp_ms": 0,
        "description": "Person seated at standing desk, looking directly at camera.",
        "setting": "home_office",
        "presenter_emotion": "enthusiastic",
        "body_language": "Forward lean, open hand gestures, direct eye contact",
        "engagement_level": "high",
        "on_screen_text": [],
        "objects": ["person", "desk", "monitor", "microphone"],
        "brands_visible": [],
        "production_markers": [],
        "lighting_quality": "good",
        "framing_quality": "excellent",
        "face_visible": true,
        "face_centered": true,
        "thumbnail_score": 0.91
      }
    ]
  },

  "content_classification": {
    "iab_categories": [
      { "id": "IAB19", "name": "Technology & Computing", "confidence": 0.97 },
      { "id": "IAB19-18", "name": "Internet Technology", "confidence": 0.91 }
    ],
    "iab_primary": "IAB19",
    "tags": ["decentralised storage", "ZiB", "encryption", "privacy", "Web3"],
    "keywords_ranked": [
      { "keyword": "decentralised storage", "relevance": 0.96 }
    ],
    "primary_language": "en",
    "contains_music": false,
    "contains_speech": true,
    "speech_clarity": "high",
    "accessibility_score": 0.87,
    "content_rating": "general",
    "suitable_for_children": false,
    "suitable_for_families": false,
    "garm": {
      "overall_floor": "floor_1",
      "suitable_for_ads": true,
      "brand_safety_score": 96,
      "categories": {
        "adult_explicit_sexual_content": { "present": false, "floor": null },
        "arms_ammunition": { "present": false, "floor": null },
        "crime_harmful_acts": { "present": false, "floor": null },
        "death_injury_military_conflict": { "present": false, "floor": null },
        "online_piracy": { "present": false, "floor": null },
        "hate_speech_acts_of_aggression": { "present": false, "floor": null },
        "obscenity_profanity": { "present": false, "floor": null },
        "illegal_drugs_tobacco_vaping": { "present": false, "floor": null },
        "spam_harmful_content": { "present": false, "floor": null },
        "terrorism": { "present": false, "floor": null },
        "debated_sensitive_social_issues": { "present": false, "floor": null }
      }
    },
    "rights": {
      "content_type": "original",
      "contains_third_party_ip": false,
      "detected_brands": [],
      "detected_music": [],
      "detected_stock_footage": false,
      "copyright_flags": []
    }
  },

  "clip_suggestions": [
    {
      "index": 0,
      "start_s": 8.4,   "end_s": 55.2,
      "start_ms": 8400, "end_ms": 55200,
      "duration_s": 46.8,
      "title": "What if your cloud storage provider can read everything you upload?",
      "hook_text": "Most people don't realise that every file you upload to S3 can be read by Amazon.",
      "reason": "Opens with provocative hook, delivers clear explanation, natural conclusion.",
      "virality_score": 0.81,
      "short_form_suitable": true,
      "vertical_crop_viable": true,
      "family_safe": true
    }
  ],

  "thumbnail_candidates": [
    {
      "rank": 1,
      "timestamp_s": 12.4, "timestamp_ms": 12400,
      "score": 0.91,
      "reason": "Clear face, direct eye contact, forward lean, good lighting.",
      "frame_object_id": "zib_abc123.thumb.0001.jpg"
    }
  ]
}

// Both are always present — pick the unit that matches your system
const segment = sidecar.transcript.segments[0];
segment.start_s   // => 0.0        (float seconds — for video players)
segment.start_ms  // => 0          (integer ms — for ad servers, caption pipelines)

const chapter = sidecar.chapters[0];
chapter.start_s   // => 142.0
chapter.start_ms  // => 142000

const clip = sidecar.clip_suggestions[0];
clip.start_ms     // => 8400       (integer — no floating point rounding to worry about)

const thumb = sidecar.thumbnail_candidates[0];
thumb.timestamp_ms // => 12400     (seek video player to this ms)

const { scene_understanding, content_classification, chapters } = sidecar;

// Auto-populate upload form
form.title       = scene_understanding.title_suggestion;
form.description = scene_understanding.description_suggestion;
form.tags        = content_classification.tags;

// Paste directly into YouTube description field
form.chapters    = sidecar.chapters_youtube_format;

// Ad placement — brand safety gate
const brandSafe = content_classification.garm.brand_safety_score > 80;
const adFloor   = content_classification.garm.overall_floor; // "floor_1"

// Place mid-roll ad at a natural viewing lull
const adMidrollAt = sidecar.scene_understanding.engagement_map
  .low_engagement_risk_segments[0]?.start_ms;

// Search indexing
const searchDoc = {
  id:          fileId,
  title:       scene_understanding.title_suggestion,
  body:        scene_understanding.executive_summary,
  transcript:  sidecar.transcript.full_text,
  tags:        content_classification.tags,
  iab:         content_classification.iab_primary,
};

// Caption delivery — segments have start_ms and end_ms ready-to-use
const captions = sidecar.transcript.segments.map(s => ({
  start: s.start_ms,
  end:   s.end_ms,
  text:  s.text,
}));

// SRT available inline — no second fetch needed
const srtContent = sidecar.subtitles.srt_inline;
const vttContent = sidecar.subtitles.vtt_inline;

// Content moderation and ad eligibility
const adEligible  = sidecar.content_classification.garm.suitable_for_ads;
const iabCategory = sidecar.content_classification.iab_primary; // e.g. "IAB19"

// Rights and IP detection
const { rights } = sidecar.content_classification;
if (rights.contains_third_party_ip || rights.copyright_flags.length > 0) {
  flagForRightsReview(fileId, rights);
}

// Find clips suitable for Reels/Shorts
const shortFormClips = sidecar.clip_suggestions.filter(c => c.short_form_suitable);

// Further filter to those safe for vertical crop (9:16)
const verticalClips = shortFormClips.filter(c => c.vertical_crop_viable);

// Each clip has everything needed to create a derivative
verticalClips.forEach(clip => {
  console.log(clip.start_ms, clip.end_ms); // trim points in ms
  console.log(clip.title);                 // suggested title
  console.log(clip.hook_text);             // opening caption
  console.log(clip.virality_score);        // 0-1 predicted virality
});

// Import sequence markers from chapters
const markers = sidecar.chapters.map(ch => ({
  name: ch.title,
  in:   ch.start_ms,
  out:  ch.end_ms,
}));

// Caption track import — SRT inline, no extra fetch
const captionTrack = sidecar.subtitles.srt_inline;

// Best poster frame — frame_object_id is a ZiB object, get CDN URL
const bestThumb = sidecar.thumbnail_candidates[0];
const thumbUrl  = zib.getCdnUrl(bestThumb.frame_object_id);

// On-screen text for auto-generated lower thirds
sidecar.scene_understanding.on_screen_text.forEach(item => {
  if (item.type === 'chyron' || item.type === 'title_card') {
    addLowerThird(item.timestamp_ms, item.text);
  }
});

const { content_classification: cc } = sidecar;

// IAB Content Taxonomy 3.0 — readable by all major DSPs without translation
const targeting = {
  iab_categories: cc.iab_categories.map(c => c.id), // ["IAB19", "IAB19-18"]
  iab_primary:    cc.iab_primary,
};

// GARM brand safety — same standard used by Snapchat, YouTube, all DSPs
const brandSafety = {
  floor:          cc.garm.overall_floor,  // "floor_1"
  score:          cc.garm.brand_safety_score,
  suitable:       cc.garm.suitable_for_ads,
  categories:     cc.garm.categories,    // 11 GARM categories, each with floor
};

// Engagement-driven ad slot selection
const adSlots = sidecar.scene_understanding.engagement_map
  .low_engagement_risk_segments
  .map(seg => ({ in_ms: seg.start_ms, out_ms: seg.end_ms, reason: seg.reason }));

POST /v1/api/compute/{objectId}
Authorization: Bearer ACCESS_KEY:SECRET_KEY
Content-Type: application/json

{
  "vision": "standard",
  "output_schema": {
    "my_title":    "{{scene_understanding.title_suggestion}}",
    "duration_ms": "{{video.duration_ms}}",
    "brand_safe":  "{{content_classification.garm.suitable_for_ads}}",
    "top_tag":     "{{content_classification.tags.0}}",
    "floor":       "{{content_classification.garm.overall_floor}}",
    "iab":         "{{content_classification.iab_primary}}"
  }
}