00001 <?php
00009 function convertLinks() {
00010 global $wgDBtype;
00011 if( $wgDBtype == 'postgres' ) {
00012 wfOut( "Links table already ok on Postgres.\n" );
00013 return;
00014 }
00015
00016 wfOut( "Converting links table to ID-ID...\n" );
00017
00018 global $wgLang, $wgDBserver, $wgDBadminuser, $wgDBadminpassword, $wgDBname;
00019 global $noKeys, $logPerformance, $fh;
00020
00021 $tuplesAdded = $numBadLinks = $curRowsRead = 0; #counters etc
00022 $totalTuplesInserted = 0; # total tuples INSERTed into links_temp
00023
00024 $reportCurReadProgress = true; #whether or not to give progress reports while reading IDs from cur table
00025 $curReadReportInterval = 1000; #number of rows between progress reports
00026
00027 $reportLinksConvProgress = true; #whether or not to give progress reports during conversion
00028 $linksConvInsertInterval = 1000; #number of rows per INSERT
00029
00030 $initialRowOffset = 0;
00031 #$finalRowOffset = 0; # not used yet; highest row number from links table to process
00032
00033 # Overwrite the old links table with the new one. If this is set to false,
00034 # the new table will be left at links_temp.
00035 $overwriteLinksTable = true;
00036
00037 # Don't create keys, and so allow duplicates in the new links table.
00038 # This gives a huge speed improvement for very large links tables which are MyISAM. (What about InnoDB?)
00039 $noKeys = false;
00040
00041
00042 $logPerformance = false; # output performance data to a file
00043 $perfLogFilename = "convLinksPerf.txt";
00044 #--------------------------------------------------------------------
00045
00046 $dbw = wfGetDB( DB_MASTER );
00047 list ($cur, $links, $links_temp, $links_backup) = $dbw->tableNamesN( 'cur', 'links', 'links_temp', 'links_backup' );
00048
00049
00050 $sql_limit = $dbw->limitResult( "SELECT l_from FROM $links", 1 );
00051 $res = $dbw->query( $sql_limit );
00052 if ( $dbw->fieldType( $res, 0 ) == "int" ) {
00053 wfOut( "Schema already converted\n" );
00054 return;
00055 }
00056
00057 $res = $dbw->query( "SELECT COUNT(*) AS count FROM $links" );
00058 $row = $dbw->fetchObject($res);
00059 $numRows = $row->count;
00060 $dbw->freeResult( $res );
00061
00062 if ( $numRows == 0 ) {
00063 wfOut( "Updating schema (no rows to convert)...\n" );
00064 createTempTable();
00065 } else {
00066 if ( $logPerformance ) { $fh = fopen ( $perfLogFilename, "w" ); }
00067 $baseTime = $startTime = getMicroTime();
00068 # Create a title -> cur_id map
00069 wfOut( "Loading IDs from $cur table...\n" );
00070 performanceLog ( "Reading $numRows rows from cur table...\n" );
00071 performanceLog ( "rows read vs seconds elapsed:\n" );
00072
00073 $dbw->bufferResults( false );
00074 $res = $dbw->query( "SELECT cur_namespace,cur_title,cur_id FROM $cur" );
00075 $ids = array();
00076
00077 while ( $row = $dbw->fetchObject( $res ) ) {
00078 $title = $row->cur_title;
00079 if ( $row->cur_namespace ) {
00080 $title = $wgLang->getNsText( $row->cur_namespace ) . ":$title";
00081 }
00082 $ids[$title] = $row->cur_id;
00083 $curRowsRead++;
00084 if ($reportCurReadProgress) {
00085 if (($curRowsRead % $curReadReportInterval) == 0) {
00086 performanceLog( $curRowsRead . " " . (getMicroTime() - $baseTime) . "\n" );
00087 wfOut( "\t$curRowsRead rows of $cur table read.\n" );
00088 }
00089 }
00090 }
00091 $dbw->freeResult( $res );
00092 $dbw->bufferResults( true );
00093 wfOut( "Finished loading IDs.\n\n" );
00094 performanceLog( "Took " . (getMicroTime() - $baseTime) . " seconds to load IDs.\n\n" );
00095 #--------------------------------------------------------------------
00096
00097 # Now, step through the links table (in chunks of $linksConvInsertInterval rows),
00098 # convert, and write to the new table.
00099 createTempTable();
00100 performanceLog( "Resetting timer.\n\n" );
00101 $baseTime = getMicroTime();
00102 wfOut( "Processing $numRows rows from $links table...\n" );
00103 performanceLog( "Processing $numRows rows from $links table...\n" );
00104 performanceLog( "rows inserted vs seconds elapsed:\n" );
00105
00106 for ($rowOffset = $initialRowOffset; $rowOffset < $numRows; $rowOffset += $linksConvInsertInterval) {
00107 $sqlRead = "SELECT * FROM $links ";
00108 $sqlRead = $dbw->limitResult($sqlRead, $linksConvInsertInterval,$rowOffset);
00109 $res = $dbw->query($sqlRead);
00110 if ( $noKeys ) {
00111 $sqlWrite = array("INSERT INTO $links_temp (l_from,l_to) VALUES ");
00112 } else {
00113 $sqlWrite = array("INSERT IGNORE INTO $links_temp (l_from,l_to) VALUES ");
00114 }
00115
00116 $tuplesAdded = 0; # no tuples added to INSERT yet
00117 while ( $row = $dbw->fetchObject($res) ) {
00118 $fromTitle = $row->l_from;
00119 if ( array_key_exists( $fromTitle, $ids ) ) { # valid title
00120 $from = $ids[$fromTitle];
00121 $to = $row->l_to;
00122 if ( $tuplesAdded != 0 ) {
00123 $sqlWrite[] = ",";
00124 }
00125 $sqlWrite[] = "($from,$to)";
00126 $tuplesAdded++;
00127 } else { # invalid title
00128 $numBadLinks++;
00129 }
00130 }
00131 $dbw->freeResult($res);
00132 #wfOut( "rowOffset: $rowOffset\ttuplesAdded: $tuplesAdded\tnumBadLinks: $numBadLinks\n" );
00133 if ( $tuplesAdded != 0 ) {
00134 if ($reportLinksConvProgress) {
00135 wfOut( "Inserting $tuplesAdded tuples into $links_temp..." );
00136 }
00137 $dbw->query( implode("",$sqlWrite) );
00138 $totalTuplesInserted += $tuplesAdded;
00139 if ($reportLinksConvProgress)
00140 wfOut( " done. Total $totalTuplesInserted tuples inserted.\n" );
00141 performanceLog( $totalTuplesInserted . " " . (getMicroTime() - $baseTime) . "\n" );
00142 }
00143 }
00144 wfOut( "$totalTuplesInserted valid titles and $numBadLinks invalid titles were processed.\n\n" );
00145 performanceLog( "$totalTuplesInserted valid titles and $numBadLinks invalid titles were processed.\n" );
00146 performanceLog( "Total execution time: " . (getMicroTime() - $startTime) . " seconds.\n" );
00147 if ( $logPerformance ) { fclose ( $fh ); }
00148 }
00149 #--------------------------------------------------------------------
00150
00151 if ( $overwriteLinksTable ) {
00152 $dbConn = Database::newFromParams( $wgDBserver, $wgDBadminuser, $wgDBadminpassword, $wgDBname );
00153 if (!($dbConn->isOpen())) {
00154 wfOut( "Opening connection to database failed.\n" );
00155 return;
00156 }
00157 # Check for existing links_backup, and delete it if it exists.
00158 wfOut( "Dropping backup links table if it exists..." );
00159 $dbConn->query( "DROP TABLE IF EXISTS $links_backup", DB_MASTER);
00160 wfOut( " done.\n" );
00161
00162 # Swap in the new table, and move old links table to links_backup
00163 wfOut( "Swapping tables '$links' to '$links_backup'; '$links_temp' to '$links'..." );
00164 $dbConn->query( "RENAME TABLE links TO $links_backup, $links_temp TO $links", DB_MASTER );
00165 wfOut( " done.\n\n" );
00166
00167 $dbConn->close();
00168 wfOut( "Conversion complete. The old table remains at $links_backup;\n" );
00169 wfOut( "delete at your leisure.\n" );
00170 } else {
00171 wfOut( "Conversion complete. The converted table is at $links_temp;\n" );
00172 wfOut( "the original links table is unchanged.\n" );
00173 }
00174 }
00175
00176 #--------------------------------------------------------------------
00177
00178 function createTempTable() {
00179 global $wgDBserver, $wgDBadminuser, $wgDBadminpassword, $wgDBname;
00180 global $noKeys;
00181 $dbConn = Database::newFromParams( $wgDBserver, $wgDBadminuser, $wgDBadminpassword, $wgDBname );
00182
00183 if (!($dbConn->isOpen())) {
00184 wfOut( "Opening connection to database failed.\n" );
00185 return;
00186 }
00187 $links_temp = $dbConn->tableName( 'links_temp' );
00188
00189 wfOut( "Dropping temporary links table if it exists..." );
00190 $dbConn->query( "DROP TABLE IF EXISTS $links_temp");
00191 wfOut( " done.\n" );
00192
00193 wfOut( "Creating temporary links table..." );
00194 if ( $noKeys ) {
00195 $dbConn->query( "CREATE TABLE $links_temp ( " .
00196 "l_from int(8) unsigned NOT NULL default '0', " .
00197 "l_to int(8) unsigned NOT NULL default '0')");
00198 } else {
00199 $dbConn->query( "CREATE TABLE $links_temp ( " .
00200 "l_from int(8) unsigned NOT NULL default '0', " .
00201 "l_to int(8) unsigned NOT NULL default '0', " .
00202 "UNIQUE KEY l_from(l_from,l_to), " .
00203 "KEY (l_to))");
00204 }
00205 wfOut( " done.\n\n" );
00206 }
00207
00208 function performanceLog( $text ) {
00209 global $logPerformance, $fh;
00210 if ( $logPerformance ) {
00211 fwrite( $fh, $text );
00212 }
00213 }
00214
00215 function getMicroTime() { # return time in seconds, with microsecond accuracy
00216 list($usec, $sec) = explode(" ", microtime());
00217 return ((float)$usec + (float)$sec);
00218 }