00001 <?php
00025 $originalDir = getcwd();
00026
00027 require_once( 'commandLine.inc' );
00028 require_once( 'backup.inc' );
00029
00037 class SevenZipStream {
00038 var $stream;
00039
00040 private function stripPath( $path ) {
00041 $prefix = 'mediawiki.compress.7z://';
00042 return substr( $path, strlen( $prefix ) );
00043 }
00044
00045 function stream_open( $path, $mode, $options, &$opened_path ) {
00046 if( $mode{0} == 'r' ) {
00047 $options = 'e -bd -so';
00048 } elseif( $mode{0} == 'w' ) {
00049 $options = 'a -bd -si';
00050 } else {
00051 return false;
00052 }
00053 $arg = wfEscapeShellArg( $this->stripPath( $path ) );
00054 $command = "7za $options $arg";
00055 if( !wfIsWindows() ) {
00056
00057 $command .= ' 2>/dev/null';
00058 }
00059 $this->stream = popen( $command, $mode );
00060 return ($this->stream !== false);
00061 }
00062
00063 function url_stat( $path, $flags ) {
00064 return stat( $this->stripPath( $path ) );
00065 }
00066
00067
00068
00069 function stream_close() {
00070 return fclose( $this->stream );
00071 }
00072
00073 function stream_flush() {
00074 return fflush( $this->stream );
00075 }
00076
00077 function stream_read( $count ) {
00078 return fread( $this->stream, $count );
00079 }
00080
00081 function stream_write( $data ) {
00082 return fwrite( $this->stream, $data );
00083 }
00084
00085 function stream_tell() {
00086 return ftell( $this->stream );
00087 }
00088
00089 function stream_eof() {
00090 return feof( $this->stream );
00091 }
00092
00093 function stream_seek( $offset, $whence ) {
00094 return fseek( $this->stream, $offset, $whence );
00095 }
00096 }
00097 stream_wrapper_register( 'mediawiki.compress.7z', 'SevenZipStream' );
00098
00102 class TextPassDumper extends BackupDumper {
00103 var $prefetch = null;
00104 var $input = "php://stdin";
00105 var $history = WikiExporter::FULL;
00106 var $fetchCount = 0;
00107 var $prefetchCount = 0;
00108
00109 var $failures = 0;
00110 var $maxFailures = 200;
00111 var $failureTimeout = 5;
00112
00113 var $php = "php";
00114 var $spawn = false;
00115 var $spawnProc = false;
00116 var $spawnWrite = false;
00117 var $spawnRead = false;
00118 var $spawnErr = false;
00119
00120 function dump() {
00121 # This shouldn't happen if on console... ;)
00122 header( 'Content-type: text/html; charset=UTF-8' );
00123
00124 # Notice messages will foul up your XML output even if they're
00125 # relatively harmless.
00126 if( ini_get( 'display_errors' ) )
00127 ini_set( 'display_errors', 'stderr' );
00128
00129 $this->initProgress( $this->history );
00130
00131 $this->db = $this->backupDb();
00132
00133 $this->egress = new ExportProgressFilter( $this->sink, $this );
00134
00135 $input = fopen( $this->input, "rt" );
00136 $result = $this->readDump( $input );
00137
00138 if( WikiError::isError( $result ) ) {
00139 wfDie( $result->getMessage() );
00140 }
00141
00142 if( $this->spawnProc ) {
00143 $this->closeSpawn();
00144 }
00145
00146 $this->report( true );
00147 }
00148
00149 function processOption( $opt, $val, $param ) {
00150 $url = $this->processFileOpt( $val, $param );
00151
00152 switch( $opt ) {
00153 case 'prefetch':
00154 global $IP;
00155 require_once "$IP/maintenance/backupPrefetch.inc";
00156 $this->prefetch = new BaseDump( $url );
00157 break;
00158 case 'stub':
00159 $this->input = $url;
00160 break;
00161 case 'current':
00162 $this->history = WikiExporter::CURRENT;
00163 break;
00164 case 'full':
00165 $this->history = WikiExporter::FULL;
00166 break;
00167 case 'spawn':
00168 $this->spawn = true;
00169 if( $val ) {
00170 $this->php = $val;
00171 }
00172 break;
00173 }
00174 }
00175
00176 function processFileOpt( $val, $param ) {
00177 switch( $val ) {
00178 case "file":
00179 return $param;
00180 case "gzip":
00181 return "compress.zlib://$param";
00182 case "bzip2":
00183 return "compress.bzip2://$param";
00184 case "7zip":
00185 return "mediawiki.compress.7z://$param";
00186 default:
00187 return $val;
00188 }
00189 }
00190
00194 function showReport() {
00195 if( !$this->prefetch ) {
00196 return parent::showReport();
00197 }
00198
00199 if( $this->reporting ) {
00200 $delta = wfTime() - $this->startTime;
00201 $now = wfTimestamp( TS_DB );
00202 if( $delta ) {
00203 $rate = $this->pageCount / $delta;
00204 $revrate = $this->revCount / $delta;
00205 $portion = $this->revCount / $this->maxCount;
00206 $eta = $this->startTime + $delta / $portion;
00207 $etats = wfTimestamp( TS_DB, intval( $eta ) );
00208 $fetchrate = 100.0 * $this->prefetchCount / $this->fetchCount;
00209 } else {
00210 $rate = '-';
00211 $revrate = '-';
00212 $etats = '-';
00213 $fetchrate = '-';
00214 }
00215 $this->progress( sprintf( "%s: %s %d pages (%0.3f/sec), %d revs (%0.3f/sec), %0.1f%% prefetched, ETA %s [max %d]",
00216 $now, wfWikiID(), $this->pageCount, $rate, $this->revCount, $revrate, $fetchrate, $etats, $this->maxCount ) );
00217 }
00218 }
00219
00220 function readDump( $input ) {
00221 $this->buffer = "";
00222 $this->openElement = false;
00223 $this->atStart = true;
00224 $this->state = "";
00225 $this->lastName = "";
00226 $this->thisPage = 0;
00227 $this->thisRev = 0;
00228
00229 $parser = xml_parser_create( "UTF-8" );
00230 xml_parser_set_option( $parser, XML_OPTION_CASE_FOLDING, false );
00231
00232 xml_set_element_handler( $parser, array( &$this, 'startElement' ), array( &$this, 'endElement' ) );
00233 xml_set_character_data_handler( $parser, array( &$this, 'characterData' ) );
00234
00235 $offset = 0;
00236 $bufferSize = 512 * 1024;
00237 do {
00238 $chunk = fread( $input, $bufferSize );
00239 if( !xml_parse( $parser, $chunk, feof( $input ) ) ) {
00240 wfDebug( "TextDumpPass::readDump encountered XML parsing error\n" );
00241 return new WikiXmlError( $parser, 'XML import parse failure', $chunk, $offset );
00242 }
00243 $offset += strlen( $chunk );
00244 } while( $chunk !== false && !feof( $input ) );
00245 xml_parser_free( $parser );
00246
00247 return true;
00248 }
00249
00250 function getText( $id ) {
00251 $this->fetchCount++;
00252 if( isset( $this->prefetch ) ) {
00253 $text = $this->prefetch->prefetch( $this->thisPage, $this->thisRev );
00254 if( $text === null ) {
00255
00256 } elseif( $text === "" ) {
00257
00258
00259 } else {
00260 $this->prefetchCount++;
00261 return $text;
00262 }
00263 }
00264 return $this->doGetText( $id );
00265 }
00266
00267 private function doGetText( $id ) {
00268 if( $this->spawn ) {
00269 return $this->getTextSpawned( $id );
00270 } else {
00271 return $this->getTextDbSafe( $id );
00272 }
00273 }
00274
00280 private function getTextDbSafe( $id ) {
00281 while( true ) {
00282 try {
00283 $text = $this->getTextDb( $id );
00284 $ex = new MWException("Graceful storage failure");
00285 } catch (DBQueryError $ex) {
00286 $text = false;
00287 }
00288 if( $text === false ) {
00289 $this->failures++;
00290 if( $this->failures > $this->maxFailures ) {
00291 throw $ex;
00292 } else {
00293 $this->progress( "Database failure $this->failures " .
00294 "of allowed $this->maxFailures for revision $id! " .
00295 "Pausing $this->failureTimeout seconds..." );
00296 sleep( $this->failureTimeout );
00297 }
00298 } else {
00299 return $text;
00300 }
00301 }
00302 }
00303
00307 private function getTextDb( $id ) {
00308 $id = intval( $id );
00309 $row = $this->db->selectRow( 'text',
00310 array( 'old_text', 'old_flags' ),
00311 array( 'old_id' => $id ),
00312 'TextPassDumper::getText' );
00313 $text = Revision::getRevisionText( $row );
00314 if( $text === false ) {
00315 return false;
00316 }
00317 $stripped = str_replace( "\r", "", $text );
00318 $normalized = UtfNormal::cleanUp( $stripped );
00319 return $normalized;
00320 }
00321
00322 private function getTextSpawned( $id ) {
00323 wfSuppressWarnings();
00324 if( !$this->spawnProc ) {
00325
00326 $this->openSpawn();
00327 }
00328 while( true ) {
00329
00330 $text = $this->getTextSpawnedOnce( $id );
00331 if( !is_string( $text ) ) {
00332 $this->progress("Database subprocess failed. Respawning...");
00333
00334 $this->closeSpawn();
00335 sleep( $this->failureTimeout );
00336 $this->openSpawn();
00337
00338 continue;
00339 }
00340 wfRestoreWarnings();
00341 return $text;
00342 }
00343 }
00344
00345 function openSpawn() {
00346 global $IP, $wgDBname;
00347
00348 $cmd = implode( " ",
00349 array_map( 'wfEscapeShellArg',
00350 array(
00351 $this->php,
00352 "$IP/maintenance/fetchText.php",
00353 $wgDBname ) ) );
00354 $spec = array(
00355 0 => array( "pipe", "r" ),
00356 1 => array( "pipe", "w" ),
00357 2 => array( "file", "/dev/null", "a" ) );
00358 $pipes = array();
00359
00360 $this->progress( "Spawning database subprocess: $cmd" );
00361 $this->spawnProc = proc_open( $cmd, $spec, $pipes );
00362 if( !$this->spawnProc ) {
00363
00364 $this->progress( "Subprocess spawn failed." );
00365 return false;
00366 }
00367 list(
00368 $this->spawnWrite,
00369 $this->spawnRead,
00370 ) = $pipes;
00371
00372 return true;
00373 }
00374
00375 private function closeSpawn() {
00376 wfSuppressWarnings();
00377 if( $this->spawnRead )
00378 fclose( $this->spawnRead );
00379 $this->spawnRead = false;
00380 if( $this->spawnWrite )
00381 fclose( $this->spawnWrite );
00382 $this->spawnWrite = false;
00383 if( $this->spawnErr )
00384 fclose( $this->spawnErr );
00385 $this->spawnErr = false;
00386 if( $this->spawnProc )
00387 pclose( $this->spawnProc );
00388 $this->spawnProc = false;
00389 wfRestoreWarnings();
00390 }
00391
00392 private function getTextSpawnedOnce( $id ) {
00393 $ok = fwrite( $this->spawnWrite, "$id\n" );
00394
00395 if( !$ok ) return false;
00396
00397 $ok = fflush( $this->spawnWrite );
00398
00399 if( !$ok ) return false;
00400
00401 $len = fgets( $this->spawnRead );
00402
00403 if( $len === false ) return false;
00404
00405 $nbytes = intval( $len );
00406 $text = "";
00407
00408
00409 while( $nbytes > strlen( $text ) ) {
00410 $buffer = fread( $this->spawnRead, $nbytes - strlen( $text ) );
00411 if( $text === false ) break;
00412 $text .= $buffer;
00413 }
00414
00415 $gotbytes = strlen( $text );
00416 if( $gotbytes != $nbytes ) {
00417 $this->progress( "Expected $nbytes bytes from database subprocess, got $gotbytes ");
00418 return false;
00419 }
00420
00421
00422 $stripped = str_replace( "\r", "", $text );
00423 $normalized = UtfNormal::cleanUp( $stripped );
00424 return $normalized;
00425 }
00426
00427 function startElement( $parser, $name, $attribs ) {
00428 $this->clearOpenElement( null );
00429 $this->lastName = $name;
00430
00431 if( $name == 'revision' ) {
00432 $this->state = $name;
00433 $this->egress->writeOpenPage( null, $this->buffer );
00434 $this->buffer = "";
00435 } elseif( $name == 'page' ) {
00436 $this->state = $name;
00437 if( $this->atStart ) {
00438 $this->egress->writeOpenStream( $this->buffer );
00439 $this->buffer = "";
00440 $this->atStart = false;
00441 }
00442 }
00443
00444 if( $name == "text" && isset( $attribs['id'] ) ) {
00445 $text = $this->getText( $attribs['id'] );
00446 $this->openElement = array( $name, array( 'xml:space' => 'preserve' ) );
00447 if( strlen( $text ) > 0 ) {
00448 $this->characterData( $parser, $text );
00449 }
00450 } else {
00451 $this->openElement = array( $name, $attribs );
00452 }
00453 }
00454
00455 function endElement( $parser, $name ) {
00456 if( $this->openElement ) {
00457 $this->clearOpenElement( "" );
00458 } else {
00459 $this->buffer .= "</$name>";
00460 }
00461
00462 if( $name == 'revision' ) {
00463 $this->egress->writeRevision( null, $this->buffer );
00464 $this->buffer = "";
00465 $this->thisRev = "";
00466 } elseif( $name == 'page' ) {
00467 $this->egress->writeClosePage( $this->buffer );
00468 $this->buffer = "";
00469 $this->thisPage = "";
00470 } elseif( $name == 'mediawiki' ) {
00471 $this->egress->writeCloseStream( $this->buffer );
00472 $this->buffer = "";
00473 }
00474 }
00475
00476 function characterData( $parser, $data ) {
00477 $this->clearOpenElement( null );
00478 if( $this->lastName == "id" ) {
00479 if( $this->state == "revision" ) {
00480 $this->thisRev .= $data;
00481 } elseif( $this->state == "page" ) {
00482 $this->thisPage .= $data;
00483 }
00484 }
00485 $this->buffer .= htmlspecialchars( $data );
00486 }
00487
00488 function clearOpenElement( $style ) {
00489 if( $this->openElement ) {
00490 $this->buffer .= Xml::element( $this->openElement[0], $this->openElement[1], $style );
00491 $this->openElement = false;
00492 }
00493 }
00494 }
00495
00496
00497 $dumper = new TextPassDumper( $argv );
00498
00499 if( true ) {
00500 $dumper->dump();
00501 } else {
00502 $dumper->progress( <<<ENDS
00503 This script postprocesses XML dumps from dumpBackup.php to add
00504 page text which was stubbed out (using --stub).
00505
00506 XML input is accepted on stdin.
00507 XML output is sent to stdout; progress reports are sent to stderr.
00508
00509 Usage: php dumpTextPass.php [<options>]
00510 Options:
00511 --stub=<type>:<file> To load a compressed stub dump instead of stdin
00512 --prefetch=<type>:<file> Use a prior dump file as a text source, to save
00513 pressure on the database.
00514 (Requires PHP 5.0+ and the XMLReader PECL extension)
00515 --quiet Don't dump status reports to stderr.
00516 --report=n Report position and speed after every n pages processed.
00517 (Default: 100)
00518 --server=h Force reading from MySQL server h
00519 --current Base ETA on number of pages in database instead of all revisions
00520 --spawn Spawn a subprocess for loading text records
00521 ENDS
00522 );
00523 }
00524
00525