diff options
Diffstat (limited to 'refresh.php')
| -rw-r--r-- | refresh.php | 437 |
1 files changed, 437 insertions, 0 deletions
diff --git a/refresh.php b/refresh.php new file mode 100644 index 0000000..ec12168 --- /dev/null +++ b/refresh.php @@ -0,0 +1,437 @@ +<?php +chdir(dirname($_SERVER["SCRIPT_FILENAME"])); + +if (isset($_SERVER["SERVER_NAME"])) { + echo "<pre>"; + ini_set("max_execution_time", "0"); + ini_set("memory_limit", "-1"); + + die("This script is launched twice a day, be patient"); +} +require "config.inc.php"; + +function addUpstream ($db, $type, $channel, $name, $state, $ver, $statename="") { + + $sql1 = "INSERT INTO `upstream` SET type='$type', name='$name'"; + $db->exec($sql1); + + $sql2 = "UPDATE `upstream` SET channel='$channel', $state='$ver'"; + if ($statename) $sql2 .= ", state='$statename'"; + $sql2 .= " WHERE type='$type' AND name='$name'"; + $nb = $db->exec($sql2); + if ($nb===false) echo date("r : ") . "SQL ERROR($sql2) = " . implode(" ", $db->errorInfo()) . "\n"; + + return $nb; +} +if ($_SERVER['argc']>1 && in_array('help', $_SERVER['argv'])) { + echo "Options in: repo owner R pear pecl old\n"; + echo "Defaults: repo owner R pear pecl\n"; + die("\n"); +} + +try { + $db = new PDO ("mysql:dbname=" . MYBASE . ";host=" . MYHOST, MYUSER, MYPASS); + + // ------------------------------------------------------------------- + // Fedora Packages + // ------------------------------------------------------------------- + echo date("r : ") . "Refreshing " . MYBASE . " database\n"; + + if ($_SERVER['argc']==1 || in_array('repo', $_SERVER['argv'])) { + if (in_array('empty', $_SERVER['argv'])) { + $res=$db->query("SELECT * FROM repo WHERE stamp IS NULL"); + } else if (in_array('old', $_SERVER['argv'])) { + $res=$db->query("SELECT * FROM repo"); + } else { + $res=$db->query("SELECT * FROM repo WHERE active=1"); + } + if (!$res) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; + else while ($row = $res->fetchObject()) { + echo date("r : ") . "REPOSITORY " . $row->main . " " . $row->sub . "\n"; + $TimRemote = 0; + $repomd = simplexml_load_file($row->url . "repodata/repomd.xml"); + if ($repomd) foreach($repomd->data as $data) if ($data->attributes()=="primary") { + $TimRemote = $data->timestamp; + $UrlRemote = $row->url . $data->location->attributes(); + } + if ($TimRemote > $row->stamp) { + echo date("r : ") . "Loading $UrlRemote\n"; + + //$fic=gzopen("primary.xml.gz", "r"); + $fic=gzopen($UrlRemote, "r"); + if ($fic) { + $txt=""; + while ($buf=gzread($fic,8196)) + $txt .= $buf; + echo date("r : ") . "Read " . strlen($txt) . " bytes\n"; + gzclose($fic); + + $primary = simplexml_load_string($txt); + echo date("r : ") . "Read " . $primary->attributes() . " packages\n"; + unset($txt); + + $sql = sprintf("DELETE FROM rpm WHERE repo_main='%s' AND repo_sub='%s'", + $row->main, $row->sub); + $nb=$db->exec($sql); + if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; + else echo date("r : ") . "Delete $nb packages\n"; + + $tot=0; + foreach ($primary->package as $package) if ($package->attributes()=='rpm'){ + $ver = $package->version->attributes(); + $sql = sprintf("INSERT INTO rpm SET repo_main='%s', repo_sub='%s', name='%s', epoch='%s', ver='%s', rel='%s', summary='%s', url='%s'", + $row->main, $row->sub, $package->name, $ver['epoch'], $ver['ver'], $ver['rel'], addslashes($package->summary), $package->url); + $nb=$db->exec($sql); + if ($nb) $tot+=$nb; + else echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; + } + echo date("r : ") . "Write $tot packages\n"; + + $sql=sprintf("UPDATE repo SET stamp=%d WHERE ID=%d", $TimRemote, $row->ID); + $nb=$db->exec($sql); + if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; + + unset($primary); + } else { + echo date("r : ") . "ERROR : can't read $UrlRemote\n"; + } + } else { + echo date("r : ") . "no update needed : $TimRemote / " . $row->stamp . "\n"; + } + } + } // If ask + + // ------------------------------------------------------------------- + // Upstream Version + // ------------------------------------------------------------------- + $sql="CREATE TABLE IF NOT EXISTS `upstream` ( + `name` varchar(100) NOT NULL, + `type` varchar(20) NOT NULL, + `channel` varchar(20) NOT NULL, + `stable` varchar(20) DEFAULT NULL, + `unstable` varchar(20) DEFAULT NULL, + `state` varchar(20) DEFAULT NULL, + PRIMARY KEY (`name`,`type`), + KEY `name` (`name`), + KEY `type` (`type`), + KEY `channel` (`channel`))"; + + if ($db->exec($sql)!==false) + echo date("r : ") . "Check table 'upstream' ok\n"; + else echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; + + // ------------------------------------------------------------------- + if ($_SERVER['argc']==1 || in_array('pecl', $_SERVER['argv'])) { + echo date("r : ") . "PECL listLatestReleases - stable\n"; + + $request = xmlrpc_encode_request("package.listLatestReleases", "stable"); + $context = stream_context_create(array('http' => array( + 'method' => "POST", + 'header' => "Content-Type: text/xml", + 'content' => $request + ))); + $file = file_get_contents("http://pecl.php.net/xmlrpc.php", false, $context) or die ("Can't file_get_contents(http://pecl.php.net/xmlrpc.php)"); + $stable = xmlrpc_decode($file); + if (xmlrpc_is_fault($stable)) { + echo date("r : ") . "ERROR xmlrpc: $stable[faultString] ($stable[faultCode])"; + } else { + $sql = sprintf("DELETE FROM upstream WHERE type='pecl' AND channel='pecl'"); + $nb=$db->exec($sql); + if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; + else echo date("r : ") . "Delete $nb packages\n"; + + $nb=0; + foreach ($stable as $name => $info) { + $rpmname="php-pecl-".str_replace("_","-",$name); + + $nb += addUpstream($db, "pecl", "pecl", $rpmname, "stable", $info["version"]); + } + echo date("r : ") . "Write $nb packages\n"; + // echo date("r : ") . "saved " . file_put_contents("cache/pecl-stable", serialize($stable)) . " bytes\n"; + } + + // ------------------------------------------------------------------- + echo date("r : ") . "PECL listLatestReleases - unstable\n"; + + $request = xmlrpc_encode_request("package.listLatestReleases", array()); + $context = stream_context_create(array('http' => array( + 'method' => "POST", + 'header' => "Content-Type: text/xml", + 'content' => $request + ))); + $file = file_get_contents("http://pecl.php.net/xmlrpc.php", false, $context) or die ("Can't file_get_contents(http://pecl.php.net/xmlrpc.php)"); + $unstable = xmlrpc_decode($file); + if (xmlrpc_is_fault($unstable)) { + echo date("r : ") . "ERROR xmlrpc: $stable[faultString] ($stable[faultCode])"; + } else { + $nb=0; + foreach ($unstable as $name => $info) { + $rpmname="php-pecl-".str_replace("_","-",$name); + + $nb += addUpstream($db, "pecl", "pecl", $rpmname, "unstable", $info["version"], $info["state"]); + } + echo date("r : ") . "Write $nb packages\n"; + //echo date("r : ") . "saved " . file_put_contents("cache/pecl-unstable", serialize($unstable)) . " bytes\n"; + } + } // if in options + + // ------------------------------------------------------------------- + if ($_SERVER['argc']==1 || in_array('pear', $_SERVER['argv'])) { + echo date("r : ") . "PEAR reading channels\n"; + $channels=array( + "pear" => "pear.php.net" + ,"doctrine" => "pear.phpdoctrine.org" + ,"ezc" => "components.ez.no" + ,"pdepend" => "pear.pdepend.org" + ,"phing" => "pear.phing.info" + ,"phpdb" => "pear.phpdb.org" + ,"phpmd" => "pear.phpmd.org" + ,"phpunit" => "pear.phpunit.de" + ,"swift" => "pear.swiftmailer.org" + ,"symphony" => "pear.symfony-project.com" + ); + try { + $nbtot=0; + + foreach ($channels as $channelname => $channelurl) { + + $channel = @simplexml_load_file("http://$channelurl/channel.xml"); + if (!$channel) throw new Exception("can't read PEAR site (channel of $channelname)"); + + $rest = $channel->servers->primary->rest->baseurl[0]; + echo date("r : ") . "PEAR reading channel=$channelname, baseurl = $rest\n"; + + $categories = @simplexml_load_file($rest."c/categories.xml"); + if (!$categories) throw new Exception("can't read PEAR site (categories)"); + + $sql = sprintf("DELETE FROM upstream WHERE type='pear' AND channel='$channelname'"); + $nb=$db->exec($sql); + if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; + else echo date("r : ") . "Delete $nb packages\n"; + + $nb=0; + if (!isset($categories->c[0])) { + echo date("r : ") . "Reading ALL\n"; // ezc only + $pitxt = @file_get_contents($rest."p/packages.xml"); + if (!$pitxt) throw new Exception("can't read PEAR site (".$rest."p/packagesinfo.xml)"); + $allpi = @simplexml_load_string($pitxt); + foreach ($allpi->p as $name) { + $pitxt = @file_get_contents($rest."r/".strtolower($name)."/allreleases.xml"); + if (!$pitxt) throw new Exception("can't read PEAR site (".$rest."r/".strtolower($name)."/allreleases.xml"); + $pi = @simplexml_load_string($pitxt); + + $rpmname1="php-".$channelname."-".str_replace("_","-",$name); + $rpmname2="php-".$channelname."-".$name; + + //echo $rpmname ."/".(string)$pi->r[0]->v."/". (string)$pi->r[0]->s."\n"; + addUpstream($db, "pear", $channelname, $rpmname1, "unstable", (string)$pi->r[0]->v, (string)$pi->r[0]->s); + addUpstream($db, "pear", $channelname, $rpmname2, "unstable", (string)$pi->r[0]->v, (string)$pi->r[0]->s); + foreach($pi->r as $rev) if ($rev->s=='stable') { + //echo $rpmname ."/".(string)$rev->v."/". (string)$rev->s."\n"; + addUpstream($db, "pear", $channelname, $rpmname1, "stable", (string)$rev->v); + addUpstream($db, "pear", $channelname, $rpmname2, "stable", (string)$rev->v); + break; + } + $nb++; + } + + } else foreach ($categories->c as $cat) { + echo date("r : ") . "Reading $cat\n"; + + $pitxt = @file_get_contents($rest."c/".urlencode($cat)."/packagesinfo.xml"); + if (!$pitxt) throw new Exception("can't read PEAR site (".$rest."c/".urlencode($cat)."/packagesinfo.xml)"); + $pitxt = "<?" . preg_replace("/<\?xml.*?>/U", "", str_replace("\r\n", "\n", substr($pitxt, 2))); + + $pi = @simplexml_load_string($pitxt); + if (!$pi) throw new Exception("can't read response ($cat)"); + foreach ($pi->pi as $ps) if (isset($ps->p->n) && isset($ps->a->r)) { + $name=(string)$ps->p->n; + + if ($channelname=='phing' && $name=='phing') { + $rpmname1="php-pear-phing"; + } else if ($channelname=='phpunit' && $name=='PHPUnit') { + $rpmname1="php-pear-PHPUnit"; + } else { + $rpmname1="php-".$channelname."-".str_replace("_","-",$name); + } + $rpmname2="php-".$channelname."-".$name; + + addUpstream($db, "pear", $channelname, $rpmname1, "unstable", (string)$ps->a->r[0]->v, (string)$ps->a->r[0]->s); + addUpstream($db, "pear", $channelname, $rpmname2, "unstable", (string)$ps->a->r[0]->v, (string)$ps->a->r[0]->s); + foreach($ps->a->r as $rev) if ($rev->s=='stable') { + addUpstream($db, "pear", $channelname, $rpmname1, "stable", (string)$rev->v); + addUpstream($db, "pear", $channelname, $rpmname2, "stable", (string)$rev->v); + break; + } + $nb++; + // echo $ps->p->n."\n"; + } + } + echo date("r : ") . "read $nb packages in $channelname\n"; + $nbtot += $nb; + } + echo date("r : ") . "read $nbtot packages in all channels\n"; + + } catch (Exception $e) { + echo date("r : ") . 'ERROR ' . $e->getMessage(), "\n"; + } + } // if in options + // ------------------------------------------------------------------- + if ($_SERVER['argc']==1 || in_array('R', $_SERVER['argv'])) { + + $repos = array( + array("name"=>"R", "state"=>"stable", "url" => "http://cran.at.r-project.org/src/contrib/PACKAGES"), + //array("name"=>"R-forge","state"=>"unstable", "url" => "http://r-forge.r-project.org/src/contrib/PACKAGES"), + array("name"=>"biocp", "state"=>"stable", "url" => "http://www.bioconductor.org/packages/release/bioc/src/contrib/PACKAGES"), + array("name"=>"biocp", "state"=>"unstable", "url" => "http://www.bioconductor.org/packages/devel/bioc/src/contrib/PACKAGES"), + array("name"=>"bioca", "state"=>"stable", "url" => "http://www.bioconductor.org/packages/release/data/annotation/src/contrib/PACKAGES"), + array("name"=>"bioca", "state"=>"unstable", "url" => "http://www.bioconductor.org/packages/devel/data/annotation/src/contrib/PACKAGES"), + array("name"=>"bioce", "state"=>"stable", "url" => "http://www.bioconductor.org/packages/release/data/experiment/src/contrib/PACKAGES"), + array("name"=>"bioce", "state"=>"unstable", "url" => "http://www.bioconductor.org/packages/devel/data/experiment/src/contrib/PACKAGES") + ); + $tottot=0; + foreach ($repos as $repo) { + echo date("r : ") . "Reading " . $repo["name"] . " (" . $repo["state"] . ")\n"; + $index = @file_get_contents($repo["url"]); + if (!$index) { + echo date("r : ") . "Can't read [" . $repo["url"] . "], skip this channel\n"; + continue; + } + if ($repo["state"]=="stable") { + $sql = sprintf("DELETE FROM upstream WHERE type='R' AND channel='%s'", $repo["name"]); + $nb=$db->exec($sql); + if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; + else echo date("r : ") . "Delete $nb packages\n"; + } + $results=array(); + if (preg_match_all('/Package: *(.*)\nVersion: *(.*)\n/i',$index,$results,PREG_SET_ORDER)) { + $tot=0; + foreach ($results as $result) { + //echo $result[1]." = ".$result[2]."\n"; + $rpmname = "R-".$result[1]; + $ver = str_replace('-', '.', $result[2]); + if (addUpstream($db, "R", $repo["name"], $rpmname, $repo["state"], $ver, ($repo["state"]=="stable"?"":"devel"))) { + $tot++; + } + } + echo date("r : ") . "Write $tot packages in this channel\n"; + $tottot += $tot; + } else { + echo date("r : ") . "No package in this channel\n"; + } + } + echo date("r : ") . "Write $tottot packages in all channels\n"; +/* + * OLD VERSION parsing the HTML pages + $repos = array( + array("name"=>"R", "state"=>"stable", "url" => "http://cran.r-project.org/web/packages/"), + array("name"=>"biocp", "state"=>"stable", "url" => "http://bioconductor.org/packages/release/bioc/"), + array("name"=>"biocp", "state"=>"unstable", "url" => "http://bioconductor.org/packages/devel/bioc/"), + array("name"=>"bioca", "state"=>"stable", "url" => "http://bioconductor.org/packages/release/data/annotation/"), + array("name"=>"bioca", "state"=>"unstable", "url" => "http://bioconductor.org/packages/devel/data/annotation/"), + array("name"=>"bioce", "state"=>"stable", "url" => "http://bioconductor.org/packages/release/data/experiment/"), + array("name"=>"bioce", "state"=>"unstable", "url" => "http://bioconductor.org/packages/devel/data/experiment/") + ); + $tottot=0; + foreach ($repos as $repo) { + echo date("r : ") . "Reading " . $repo["name"] . " (" . $repo["state"] . ")\n"; + $index = @file_get_contents($repo["url"]); + if (!$index) { + echo date("r : ") . "Can't read [" . $repo["url"] . "], skip this channel\n"; + continue; + } + if ($repo["state"]=="stable") { + $sql = sprintf("DELETE FROM upstream WHERE type='R' AND channel='%s'", $repo["name"]); + $nb=$db->exec($sql); + if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; + else echo date("r : ") . "Delete $nb packages\n"; + } + if ($repo["name"] == "R") { + $pos = strpos($index, 'ID="available-packages'); + $pat1 = '/<A HREF="(\.\.\/\.\.\/web\/.*)">(.*)<\/A>/iU'; + } else { + $pos = 0; + $pat1 = '/<a href="(.*)">(.*)<\/A>/iu'; + } + $results=array(); + if (preg_match_all($pat1,$index,$results,PREG_SET_ORDER,$pos)) { + $tot=0; + foreach ($results as $result) { + $page = @file_get_contents($repo["url"].$result[1]); + $name = $result[2]; + $rpmname = "R-".$result[2]; + $ver=array(); + if (!$page) { + echo date("r : ") . "Can't read [" . $repo["url"].$result[1] . "], skip this package\n"; + + } else if (preg_match("/".$name."_((\d|-|\.)*)\.tar.gz/", $page, $ver)) { + + $ver = str_replace('-', '.', $ver[1]); + $nb = addUpstream($db, "R", $repo["name"], $rpmname, $repo["state"], $ver, ($repo["state"]=="stable"?"":"devel")); + + } + + if ($nb) $tot+=$nb; + + } + echo date("r : ") . "Write $tot packages in this channel\n"; + $tottot += $tot; + } // Each package + } // Each repo + echo date("r : ") . "Write $tottot packages in all channels\n"; + */ + } // If R in options + + // ------------------------------------------------------------------- + // Package Owners from pkgdb (thanks Smootherfrog) + // ------------------------------------------------------------------- + if ($_SERVER['argc']==1 || in_array('owner', $_SERVER['argv'])) { + + $sql="CREATE TABLE IF NOT EXISTS `acls` ( + `ID` int(11) NOT NULL AUTO_INCREMENT, + `collection` varchar(100) NOT NULL, + `name` varchar(100) NOT NULL, + `summary` varchar(200) NOT NULL, + `owner` varchar(50) DEFAULT NULL, + `qa` varchar(50) DEFAULT NULL, + `cc` varchar(100) DEFAULT NULL, + PRIMARY KEY (`ID`), + KEY `name` (`name`), + KEY `collection` (`collection`), + KEY `owner` (`owner`)) ENGINE=MyISAM"; + + if ($db->exec($sql)!==false) + echo date("r : ") . "Check table 'acls' ok\n"; + else echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; + + $fic=fopen("https://admin.fedoraproject.org/pkgdb/acls/bugzilla?tg_format=plain", "r"); + if (!$fic) { + echo date("r : ") . "ERROR reading pkgdb\n"; + } else { + $sql="TRUNCATE `acls`"; + $nb=$db->exec($sql); + if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; + else echo date("r : ") . "Delete $nb packages\n"; + + for ($tot=0 ; $line=fgetcsv($fic,1024,'|') ; ) if (count($line)>5 && substr($line[0],0,1)!='#') { + for ($i=0; $i<6; $i++) $line[$i]=trim($line[$i]); + $sql=sprintf("INSERT INTO `acls` SET collection='%s', name='%s', summary='%s'", $line[0], $line[1], addslashes($line[2])); + if (!empty($line[3])) $sql .= sprintf(", owner='%s'", $line[3]); + if (!empty($line[4])) $sql .= sprintf(", qa='%s'", $line[4]); + if (!empty($line[5])) $sql .= sprintf(", cc='%s'", $line[5]); + $nb=$db->exec($sql); + if ($nb) $tot+=$nb; + else echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; + } + fclose($fic); + echo date("r : ") . "wrote $tot package's owner\n"; + } + } // If in options + +} +catch(PDOException $e) { + printf("%s ERREUR : %s\n", date("r"), $e->getMessage()); +} + +if (isset($_SERVER["SERVER_NAME"])) echo "</pre>"; +?> |