summaryrefslogtreecommitdiffstats
path: root/refresh.php
diff options
context:
space:
mode:
Diffstat (limited to 'refresh.php')
-rw-r--r--refresh.php437
1 files changed, 437 insertions, 0 deletions
diff --git a/refresh.php b/refresh.php
new file mode 100644
index 0000000..ec12168
--- /dev/null
+++ b/refresh.php
@@ -0,0 +1,437 @@
+<?php
+chdir(dirname($_SERVER["SCRIPT_FILENAME"]));
+
+if (isset($_SERVER["SERVER_NAME"])) {
+ echo "<pre>";
+ ini_set("max_execution_time", "0");
+ ini_set("memory_limit", "-1");
+
+ die("This script is launched twice a day, be patient");
+}
+require "config.inc.php";
+
+function addUpstream ($db, $type, $channel, $name, $state, $ver, $statename="") {
+
+ $sql1 = "INSERT INTO `upstream` SET type='$type', name='$name'";
+ $db->exec($sql1);
+
+ $sql2 = "UPDATE `upstream` SET channel='$channel', $state='$ver'";
+ if ($statename) $sql2 .= ", state='$statename'";
+ $sql2 .= " WHERE type='$type' AND name='$name'";
+ $nb = $db->exec($sql2);
+ if ($nb===false) echo date("r : ") . "SQL ERROR($sql2) = " . implode(" ", $db->errorInfo()) . "\n";
+
+ return $nb;
+}
+if ($_SERVER['argc']>1 && in_array('help', $_SERVER['argv'])) {
+ echo "Options in: repo owner R pear pecl old\n";
+ echo "Defaults: repo owner R pear pecl\n";
+ die("\n");
+}
+
+try {
+ $db = new PDO ("mysql:dbname=" . MYBASE . ";host=" . MYHOST, MYUSER, MYPASS);
+
+ // -------------------------------------------------------------------
+ // Fedora Packages
+ // -------------------------------------------------------------------
+ echo date("r : ") . "Refreshing " . MYBASE . " database\n";
+
+ if ($_SERVER['argc']==1 || in_array('repo', $_SERVER['argv'])) {
+ if (in_array('empty', $_SERVER['argv'])) {
+ $res=$db->query("SELECT * FROM repo WHERE stamp IS NULL");
+ } else if (in_array('old', $_SERVER['argv'])) {
+ $res=$db->query("SELECT * FROM repo");
+ } else {
+ $res=$db->query("SELECT * FROM repo WHERE active=1");
+ }
+ if (!$res) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";
+ else while ($row = $res->fetchObject()) {
+ echo date("r : ") . "REPOSITORY " . $row->main . " " . $row->sub . "\n";
+ $TimRemote = 0;
+ $repomd = simplexml_load_file($row->url . "repodata/repomd.xml");
+ if ($repomd) foreach($repomd->data as $data) if ($data->attributes()=="primary") {
+ $TimRemote = $data->timestamp;
+ $UrlRemote = $row->url . $data->location->attributes();
+ }
+ if ($TimRemote > $row->stamp) {
+ echo date("r : ") . "Loading $UrlRemote\n";
+
+ //$fic=gzopen("primary.xml.gz", "r");
+ $fic=gzopen($UrlRemote, "r");
+ if ($fic) {
+ $txt="";
+ while ($buf=gzread($fic,8196))
+ $txt .= $buf;
+ echo date("r : ") . "Read " . strlen($txt) . " bytes\n";
+ gzclose($fic);
+
+ $primary = simplexml_load_string($txt);
+ echo date("r : ") . "Read " . $primary->attributes() . " packages\n";
+ unset($txt);
+
+ $sql = sprintf("DELETE FROM rpm WHERE repo_main='%s' AND repo_sub='%s'",
+ $row->main, $row->sub);
+ $nb=$db->exec($sql);
+ if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";
+ else echo date("r : ") . "Delete $nb packages\n";
+
+ $tot=0;
+ foreach ($primary->package as $package) if ($package->attributes()=='rpm'){
+ $ver = $package->version->attributes();
+ $sql = sprintf("INSERT INTO rpm SET repo_main='%s', repo_sub='%s', name='%s', epoch='%s', ver='%s', rel='%s', summary='%s', url='%s'",
+ $row->main, $row->sub, $package->name, $ver['epoch'], $ver['ver'], $ver['rel'], addslashes($package->summary), $package->url);
+ $nb=$db->exec($sql);
+ if ($nb) $tot+=$nb;
+ else echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";
+ }
+ echo date("r : ") . "Write $tot packages\n";
+
+ $sql=sprintf("UPDATE repo SET stamp=%d WHERE ID=%d", $TimRemote, $row->ID);
+ $nb=$db->exec($sql);
+ if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";
+
+ unset($primary);
+ } else {
+ echo date("r : ") . "ERROR : can't read $UrlRemote\n";
+ }
+ } else {
+ echo date("r : ") . "no update needed : $TimRemote / " . $row->stamp . "\n";
+ }
+ }
+ } // If ask
+
+ // -------------------------------------------------------------------
+ // Upstream Version
+ // -------------------------------------------------------------------
+ $sql="CREATE TABLE IF NOT EXISTS `upstream` (
+ `name` varchar(100) NOT NULL,
+ `type` varchar(20) NOT NULL,
+ `channel` varchar(20) NOT NULL,
+ `stable` varchar(20) DEFAULT NULL,
+ `unstable` varchar(20) DEFAULT NULL,
+ `state` varchar(20) DEFAULT NULL,
+ PRIMARY KEY (`name`,`type`),
+ KEY `name` (`name`),
+ KEY `type` (`type`),
+ KEY `channel` (`channel`))";
+
+ if ($db->exec($sql)!==false)
+ echo date("r : ") . "Check table 'upstream' ok\n";
+ else echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";
+
+ // -------------------------------------------------------------------
+ if ($_SERVER['argc']==1 || in_array('pecl', $_SERVER['argv'])) {
+ echo date("r : ") . "PECL listLatestReleases - stable\n";
+
+ $request = xmlrpc_encode_request("package.listLatestReleases", "stable");
+ $context = stream_context_create(array('http' => array(
+ 'method' => "POST",
+ 'header' => "Content-Type: text/xml",
+ 'content' => $request
+ )));
+ $file = file_get_contents("http://pecl.php.net/xmlrpc.php", false, $context) or die ("Can't file_get_contents(http://pecl.php.net/xmlrpc.php)");
+ $stable = xmlrpc_decode($file);
+ if (xmlrpc_is_fault($stable)) {
+ echo date("r : ") . "ERROR xmlrpc: $stable[faultString] ($stable[faultCode])";
+ } else {
+ $sql = sprintf("DELETE FROM upstream WHERE type='pecl' AND channel='pecl'");
+ $nb=$db->exec($sql);
+ if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";
+ else echo date("r : ") . "Delete $nb packages\n";
+
+ $nb=0;
+ foreach ($stable as $name => $info) {
+ $rpmname="php-pecl-".str_replace("_","-",$name);
+
+ $nb += addUpstream($db, "pecl", "pecl", $rpmname, "stable", $info["version"]);
+ }
+ echo date("r : ") . "Write $nb packages\n";
+ // echo date("r : ") . "saved " . file_put_contents("cache/pecl-stable", serialize($stable)) . " bytes\n";
+ }
+
+ // -------------------------------------------------------------------
+ echo date("r : ") . "PECL listLatestReleases - unstable\n";
+
+ $request = xmlrpc_encode_request("package.listLatestReleases", array());
+ $context = stream_context_create(array('http' => array(
+ 'method' => "POST",
+ 'header' => "Content-Type: text/xml",
+ 'content' => $request
+ )));
+ $file = file_get_contents("http://pecl.php.net/xmlrpc.php", false, $context) or die ("Can't file_get_contents(http://pecl.php.net/xmlrpc.php)");
+ $unstable = xmlrpc_decode($file);
+ if (xmlrpc_is_fault($unstable)) {
+ echo date("r : ") . "ERROR xmlrpc: $stable[faultString] ($stable[faultCode])";
+ } else {
+ $nb=0;
+ foreach ($unstable as $name => $info) {
+ $rpmname="php-pecl-".str_replace("_","-",$name);
+
+ $nb += addUpstream($db, "pecl", "pecl", $rpmname, "unstable", $info["version"], $info["state"]);
+ }
+ echo date("r : ") . "Write $nb packages\n";
+ //echo date("r : ") . "saved " . file_put_contents("cache/pecl-unstable", serialize($unstable)) . " bytes\n";
+ }
+ } // if in options
+
+ // -------------------------------------------------------------------
+ if ($_SERVER['argc']==1 || in_array('pear', $_SERVER['argv'])) {
+ echo date("r : ") . "PEAR reading channels\n";
+ $channels=array(
+ "pear" => "pear.php.net"
+ ,"doctrine" => "pear.phpdoctrine.org"
+ ,"ezc" => "components.ez.no"
+ ,"pdepend" => "pear.pdepend.org"
+ ,"phing" => "pear.phing.info"
+ ,"phpdb" => "pear.phpdb.org"
+ ,"phpmd" => "pear.phpmd.org"
+ ,"phpunit" => "pear.phpunit.de"
+ ,"swift" => "pear.swiftmailer.org"
+ ,"symphony" => "pear.symfony-project.com"
+ );
+ try {
+ $nbtot=0;
+
+ foreach ($channels as $channelname => $channelurl) {
+
+ $channel = @simplexml_load_file("http://$channelurl/channel.xml");
+ if (!$channel) throw new Exception("can't read PEAR site (channel of $channelname)");
+
+ $rest = $channel->servers->primary->rest->baseurl[0];
+ echo date("r : ") . "PEAR reading channel=$channelname, baseurl = $rest\n";
+
+ $categories = @simplexml_load_file($rest."c/categories.xml");
+ if (!$categories) throw new Exception("can't read PEAR site (categories)");
+
+ $sql = sprintf("DELETE FROM upstream WHERE type='pear' AND channel='$channelname'");
+ $nb=$db->exec($sql);
+ if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";
+ else echo date("r : ") . "Delete $nb packages\n";
+
+ $nb=0;
+ if (!isset($categories->c[0])) {
+ echo date("r : ") . "Reading ALL\n"; // ezc only
+ $pitxt = @file_get_contents($rest."p/packages.xml");
+ if (!$pitxt) throw new Exception("can't read PEAR site (".$rest."p/packagesinfo.xml)");
+ $allpi = @simplexml_load_string($pitxt);
+ foreach ($allpi->p as $name) {
+ $pitxt = @file_get_contents($rest."r/".strtolower($name)."/allreleases.xml");
+ if (!$pitxt) throw new Exception("can't read PEAR site (".$rest."r/".strtolower($name)."/allreleases.xml");
+ $pi = @simplexml_load_string($pitxt);
+
+ $rpmname1="php-".$channelname."-".str_replace("_","-",$name);
+ $rpmname2="php-".$channelname."-".$name;
+
+ //echo $rpmname ."/".(string)$pi->r[0]->v."/". (string)$pi->r[0]->s."\n";
+ addUpstream($db, "pear", $channelname, $rpmname1, "unstable", (string)$pi->r[0]->v, (string)$pi->r[0]->s);
+ addUpstream($db, "pear", $channelname, $rpmname2, "unstable", (string)$pi->r[0]->v, (string)$pi->r[0]->s);
+ foreach($pi->r as $rev) if ($rev->s=='stable') {
+ //echo $rpmname ."/".(string)$rev->v."/". (string)$rev->s."\n";
+ addUpstream($db, "pear", $channelname, $rpmname1, "stable", (string)$rev->v);
+ addUpstream($db, "pear", $channelname, $rpmname2, "stable", (string)$rev->v);
+ break;
+ }
+ $nb++;
+ }
+
+ } else foreach ($categories->c as $cat) {
+ echo date("r : ") . "Reading $cat\n";
+
+ $pitxt = @file_get_contents($rest."c/".urlencode($cat)."/packagesinfo.xml");
+ if (!$pitxt) throw new Exception("can't read PEAR site (".$rest."c/".urlencode($cat)."/packagesinfo.xml)");
+ $pitxt = "<?" . preg_replace("/<\?xml.*?>/U", "", str_replace("\r\n", "\n", substr($pitxt, 2)));
+
+ $pi = @simplexml_load_string($pitxt);
+ if (!$pi) throw new Exception("can't read response ($cat)");
+ foreach ($pi->pi as $ps) if (isset($ps->p->n) && isset($ps->a->r)) {
+ $name=(string)$ps->p->n;
+
+ if ($channelname=='phing' && $name=='phing') {
+ $rpmname1="php-pear-phing";
+ } else if ($channelname=='phpunit' && $name=='PHPUnit') {
+ $rpmname1="php-pear-PHPUnit";
+ } else {
+ $rpmname1="php-".$channelname."-".str_replace("_","-",$name);
+ }
+ $rpmname2="php-".$channelname."-".$name;
+
+ addUpstream($db, "pear", $channelname, $rpmname1, "unstable", (string)$ps->a->r[0]->v, (string)$ps->a->r[0]->s);
+ addUpstream($db, "pear", $channelname, $rpmname2, "unstable", (string)$ps->a->r[0]->v, (string)$ps->a->r[0]->s);
+ foreach($ps->a->r as $rev) if ($rev->s=='stable') {
+ addUpstream($db, "pear", $channelname, $rpmname1, "stable", (string)$rev->v);
+ addUpstream($db, "pear", $channelname, $rpmname2, "stable", (string)$rev->v);
+ break;
+ }
+ $nb++;
+ // echo $ps->p->n."\n";
+ }
+ }
+ echo date("r : ") . "read $nb packages in $channelname\n";
+ $nbtot += $nb;
+ }
+ echo date("r : ") . "read $nbtot packages in all channels\n";
+
+ } catch (Exception $e) {
+ echo date("r : ") . 'ERROR ' . $e->getMessage(), "\n";
+ }
+ } // if in options
+ // -------------------------------------------------------------------
+ if ($_SERVER['argc']==1 || in_array('R', $_SERVER['argv'])) {
+
+ $repos = array(
+ array("name"=>"R", "state"=>"stable", "url" => "http://cran.at.r-project.org/src/contrib/PACKAGES"),
+ //array("name"=>"R-forge","state"=>"unstable", "url" => "http://r-forge.r-project.org/src/contrib/PACKAGES"),
+ array("name"=>"biocp", "state"=>"stable", "url" => "http://www.bioconductor.org/packages/release/bioc/src/contrib/PACKAGES"),
+ array("name"=>"biocp", "state"=>"unstable", "url" => "http://www.bioconductor.org/packages/devel/bioc/src/contrib/PACKAGES"),
+ array("name"=>"bioca", "state"=>"stable", "url" => "http://www.bioconductor.org/packages/release/data/annotation/src/contrib/PACKAGES"),
+ array("name"=>"bioca", "state"=>"unstable", "url" => "http://www.bioconductor.org/packages/devel/data/annotation/src/contrib/PACKAGES"),
+ array("name"=>"bioce", "state"=>"stable", "url" => "http://www.bioconductor.org/packages/release/data/experiment/src/contrib/PACKAGES"),
+ array("name"=>"bioce", "state"=>"unstable", "url" => "http://www.bioconductor.org/packages/devel/data/experiment/src/contrib/PACKAGES")
+ );
+ $tottot=0;
+ foreach ($repos as $repo) {
+ echo date("r : ") . "Reading " . $repo["name"] . " (" . $repo["state"] . ")\n";
+ $index = @file_get_contents($repo["url"]);
+ if (!$index) {
+ echo date("r : ") . "Can't read [" . $repo["url"] . "], skip this channel\n";
+ continue;
+ }
+ if ($repo["state"]=="stable") {
+ $sql = sprintf("DELETE FROM upstream WHERE type='R' AND channel='%s'", $repo["name"]);
+ $nb=$db->exec($sql);
+ if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";
+ else echo date("r : ") . "Delete $nb packages\n";
+ }
+ $results=array();
+ if (preg_match_all('/Package: *(.*)\nVersion: *(.*)\n/i',$index,$results,PREG_SET_ORDER)) {
+ $tot=0;
+ foreach ($results as $result) {
+ //echo $result[1]." = ".$result[2]."\n";
+ $rpmname = "R-".$result[1];
+ $ver = str_replace('-', '.', $result[2]);
+ if (addUpstream($db, "R", $repo["name"], $rpmname, $repo["state"], $ver, ($repo["state"]=="stable"?"":"devel"))) {
+ $tot++;
+ }
+ }
+ echo date("r : ") . "Write $tot packages in this channel\n";
+ $tottot += $tot;
+ } else {
+ echo date("r : ") . "No package in this channel\n";
+ }
+ }
+ echo date("r : ") . "Write $tottot packages in all channels\n";
+/*
+ * OLD VERSION parsing the HTML pages
+ $repos = array(
+ array("name"=>"R", "state"=>"stable", "url" => "http://cran.r-project.org/web/packages/"),
+ array("name"=>"biocp", "state"=>"stable", "url" => "http://bioconductor.org/packages/release/bioc/"),
+ array("name"=>"biocp", "state"=>"unstable", "url" => "http://bioconductor.org/packages/devel/bioc/"),
+ array("name"=>"bioca", "state"=>"stable", "url" => "http://bioconductor.org/packages/release/data/annotation/"),
+ array("name"=>"bioca", "state"=>"unstable", "url" => "http://bioconductor.org/packages/devel/data/annotation/"),
+ array("name"=>"bioce", "state"=>"stable", "url" => "http://bioconductor.org/packages/release/data/experiment/"),
+ array("name"=>"bioce", "state"=>"unstable", "url" => "http://bioconductor.org/packages/devel/data/experiment/")
+ );
+ $tottot=0;
+ foreach ($repos as $repo) {
+ echo date("r : ") . "Reading " . $repo["name"] . " (" . $repo["state"] . ")\n";
+ $index = @file_get_contents($repo["url"]);
+ if (!$index) {
+ echo date("r : ") . "Can't read [" . $repo["url"] . "], skip this channel\n";
+ continue;
+ }
+ if ($repo["state"]=="stable") {
+ $sql = sprintf("DELETE FROM upstream WHERE type='R' AND channel='%s'", $repo["name"]);
+ $nb=$db->exec($sql);
+ if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";
+ else echo date("r : ") . "Delete $nb packages\n";
+ }
+ if ($repo["name"] == "R") {
+ $pos = strpos($index, 'ID="available-packages');
+ $pat1 = '/<A HREF="(\.\.\/\.\.\/web\/.*)">(.*)<\/A>/iU';
+ } else {
+ $pos = 0;
+ $pat1 = '/<a href="(.*)">(.*)<\/A>/iu';
+ }
+ $results=array();
+ if (preg_match_all($pat1,$index,$results,PREG_SET_ORDER,$pos)) {
+ $tot=0;
+ foreach ($results as $result) {
+ $page = @file_get_contents($repo["url"].$result[1]);
+ $name = $result[2];
+ $rpmname = "R-".$result[2];
+ $ver=array();
+ if (!$page) {
+ echo date("r : ") . "Can't read [" . $repo["url"].$result[1] . "], skip this package\n";
+
+ } else if (preg_match("/".$name."_((\d|-|\.)*)\.tar.gz/", $page, $ver)) {
+
+ $ver = str_replace('-', '.', $ver[1]);
+ $nb = addUpstream($db, "R", $repo["name"], $rpmname, $repo["state"], $ver, ($repo["state"]=="stable"?"":"devel"));
+
+ }
+
+ if ($nb) $tot+=$nb;
+
+ }
+ echo date("r : ") . "Write $tot packages in this channel\n";
+ $tottot += $tot;
+ } // Each package
+ } // Each repo
+ echo date("r : ") . "Write $tottot packages in all channels\n";
+ */
+ } // If R in options
+
+ // -------------------------------------------------------------------
+ // Package Owners from pkgdb (thanks Smootherfrog)
+ // -------------------------------------------------------------------
+ if ($_SERVER['argc']==1 || in_array('owner', $_SERVER['argv'])) {
+
+ $sql="CREATE TABLE IF NOT EXISTS `acls` (
+ `ID` int(11) NOT NULL AUTO_INCREMENT,
+ `collection` varchar(100) NOT NULL,
+ `name` varchar(100) NOT NULL,
+ `summary` varchar(200) NOT NULL,
+ `owner` varchar(50) DEFAULT NULL,
+ `qa` varchar(50) DEFAULT NULL,
+ `cc` varchar(100) DEFAULT NULL,
+ PRIMARY KEY (`ID`),
+ KEY `name` (`name`),
+ KEY `collection` (`collection`),
+ KEY `owner` (`owner`)) ENGINE=MyISAM";
+
+ if ($db->exec($sql)!==false)
+ echo date("r : ") . "Check table 'acls' ok\n";
+ else echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";
+
+ $fic=fopen("https://admin.fedoraproject.org/pkgdb/acls/bugzilla?tg_format=plain", "r");
+ if (!$fic) {
+ echo date("r : ") . "ERROR reading pkgdb\n";
+ } else {
+ $sql="TRUNCATE `acls`";
+ $nb=$db->exec($sql);
+ if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";
+ else echo date("r : ") . "Delete $nb packages\n";
+
+ for ($tot=0 ; $line=fgetcsv($fic,1024,'|') ; ) if (count($line)>5 && substr($line[0],0,1)!='#') {
+ for ($i=0; $i<6; $i++) $line[$i]=trim($line[$i]);
+ $sql=sprintf("INSERT INTO `acls` SET collection='%s', name='%s', summary='%s'", $line[0], $line[1], addslashes($line[2]));
+ if (!empty($line[3])) $sql .= sprintf(", owner='%s'", $line[3]);
+ if (!empty($line[4])) $sql .= sprintf(", qa='%s'", $line[4]);
+ if (!empty($line[5])) $sql .= sprintf(", cc='%s'", $line[5]);
+ $nb=$db->exec($sql);
+ if ($nb) $tot+=$nb;
+ else echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";
+ }
+ fclose($fic);
+ echo date("r : ") . "wrote $tot package's owner\n";
+ }
+ } // If in options
+
+}
+catch(PDOException $e) {
+ printf("%s ERREUR : %s\n", date("r"), $e->getMessage());
+}
+
+if (isset($_SERVER["SERVER_NAME"])) echo "</pre>";
+?>