<?php
 if(php_sapi_name() != "cli")
  exit("This script is intended to be run from the CLI\n");

 $TRACKERSFILE = "trackers.json";
 $HASHFIELD = "hash"; // row field used as the dedup key
 $MAXPAGES = 100000; // safety guard against a runaway pagination loop

 // trackers to scrape, keyed by id, with the json api pagination knobs
 $TRACKERS = [
  "yts" => ["pageparam" => "page", "startpage" => 1]
 ];

 if(!file_exists($TRACKERSFILE))
  exit("Run from the trackers dir ($TRACKERSFILE not found in CWD)\n");

 // resolve config values from the definition settings defaults and any stored credentials
 function resolveconfig($def) {
  $config = [];
  if(isset($def["settings"]))
   foreach($def["settings"] as $s) {
    if(!isset($s["name"]))
     continue;
    $config[$s["name"]] = isset($s["default"]) ? $s["default"] : "";
   }
  if(isset($def["credentials"]))
   foreach($def["credentials"] as $k => $v)
    if($v !== "")
     $config[$k] = $v;
  return $config;
 }

 // substitute {{ .Config.key }} references using the resolved config
 function tmpl($str, $config) {
  return preg_replace_callback('/\{\{\s*\.Config\.(\w+)\s*\}\}/', function($m) use ($config) {
   return isset($config[$m[1]]) ? $config[$m[1]] : "";
  }, $str);
 }

 // walk a dotted path into a decoded json structure
 function dotget($node, $path) {
  foreach(explode(".", $path) as $key) {
   if(!is_array($node) || !array_key_exists($key, $node))
    return null;
   $node = $node[$key];
  }
  return $node;
 }

 // http get with a client user agent, returns [code, body]
 function httpget($url) {
  $ch = curl_init($url);
  curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
  curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
  curl_setopt($ch, CURLOPT_USERAGENT, "curl/8.5.0");
  curl_setopt($ch, CURLOPT_TIMEOUT, 60);
  $body = curl_exec($ch);
  $code = curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
  curl_close($ch);
  return [$code, $body];
 }

 // flatten one parent object into rows, one row per element of its attribute array
 // parent scalars carry over, scalar arrays join with |, on a name collision the parent value moves to movie_<key>
 function flattenrows($parent, $attribute) {
  $base = [];
  foreach($parent as $k => $v) {
   if($k === $attribute)
    continue;
   if(is_scalar($v))
    $base[$k] = $v;
   else if(is_array($v) && $v === array_filter($v, "is_scalar"))
    $base[$k] = implode("|", $v);
  }
  $rows = [];
  if(!isset($parent[$attribute]) || !is_array($parent[$attribute]))
   return $rows;
  foreach($parent[$attribute] as $item) {
   if(!is_array($item))
    continue;
   $row = $base;
   foreach($item as $k => $v) {
    if(!is_scalar($v))
     continue;
    if(array_key_exists($k, $base))
     $row["movie_".$k] = $base[$k];
    $row[$k] = $v;
   }
   $rows[] = $row;
  }
  return $rows;
 }

 $trackers = json_decode(file_get_contents($TRACKERSFILE), true);
 if(!is_array($trackers))
  exit("Cannot read $TRACKERSFILE\n");

 $only = isset($argv[1]) ? $argv[1] : "";

 foreach($TRACKERS as $id => $knobs) {
  if($only !== "" && $id !== $only)
   continue;

  echo "Tracker: $id\n";
  $firsttimestart = microtime(true);

  if(!isset($trackers[$id])) {
   echo "Not found in $TRACKERSFILE, skipping\n\n";
   continue;
  }
  $def = $trackers[$id];

  if(!isset($def["search"]["paths"][0]["path"])) {
   echo "No search path, skipping\n\n";
   continue;
  }
  if(($def["search"]["paths"][0]["response"]["type"] ?? "") !== "json") {
   echo "Not a JSON tracker, skipping\n\n";
   continue;
  }

  $config = resolveconfig($def);
  $baseurl = tmpl($def["search"]["paths"][0]["path"], $config);

  // resolve the query inputs, blanking any keyword or query dependent template
  $query = [];
  foreach(($def["search"]["inputs"] ?? []) as $k => $v) {
   $v = tmpl((string)$v, $config);
   if(strpos($v, "{{") !== false)
    $v = "";
   $query[$k] = $v;
  }

  $rowspath = $def["search"]["rows"]["selector"];
  $attribute = $def["search"]["rows"]["attribute"] ?? "";
  $delay = isset($def["requestDelay"]) ? (float)$def["requestDelay"] : 0;
  $pageparam = $knobs["pageparam"];

  // load the known hashes and reuse the header from the existing csv
  $csv = "$id.csv";
  $header = null;
  $known = [];
  if(file_exists($csv) && filesize($csv) > 0) {
   $fh = fopen($csv, "r");
   $head = fgetcsv($fh, null, ",", "\"", "");
   if($head !== false) {
    $header = $head;
    $hi = array_search($HASHFIELD, $header, true);
    while(($line = fgetcsv($fh, null, ",", "\"", "")) !== false)
     if($hi !== false && isset($line[$hi]) && $line[$hi] != "")
      $known[$line[$hi]] = true;
   }
   fclose($fh);
  }
  $existing = count($known);

  // append new torrents, the header is reused or fixed by the first row of a fresh file
  $out = fopen($csv, "a");
  $added = 0;

  $page = $knobs["startpage"];
  $stop = false;
  while(!$stop && ($page - $knobs["startpage"]) < $MAXPAGES) {
   $url = $baseurl."?".http_build_query(array_merge($query, [$pageparam => $page]));
   [$code, $body] = httpget($url);
   if($code != 200) {
    echo "Page $page: HTTP $code, stopping\n";
    break;
   }

   $container = dotget(json_decode($body, true), $rowspath);
   if(!is_array($container) || count($container) == 0)
    break;

   $n = 0;
   foreach($container as $parent) {
    if(!is_array($parent))
     continue;
    foreach(flattenrows($parent, $attribute) as $row) {
     $key = isset($row[$HASHFIELD]) ? $row[$HASHFIELD] : "";
     if($key !== "" && isset($known[$key]))
      continue;
     if($key !== "")
      $known[$key] = true;
     if($header === null) {
      $header = array_keys($row);
      fputcsv($out, $header, ",", "\"", "");
     }
     $line = [];
     foreach($header as $col)
      $line[] = isset($row[$col]) ? $row[$col] : "";
     fputcsv($out, $line, ",", "\"", "");
     $n++;
     $added++;
    }
   }
   fflush($out);
   echo "Page $page: +$n\n";

   if($n == 0)
    $stop = true;

   $page++;
   if(!$stop && $delay > 0)
    usleep((int)($delay * 1000000));
  }

  fclose($out);
  echo "Existing: $existing\n";
  echo "Added: $added\n";
  echo "Output: $csv (".round(filesize($csv) / 1000)." KB)\n";

  $exectime = round((microtime(true) - $firsttimestart) * 1000);
  echo "Execution time: $exectime ms\n\n";
 }
?>