parselogs.php

<?php
//This has very little documentation, and is a hobbled togeather mess where speed to code and preformance of the code were placed before code readability or maintainability. Only works on linux. Requires gzip and gunzip shell commands as well as the php extentions.

//	edit the server array below to your server(s) 
//	symlink server-gamedata/servername to the static folder of tgs3 (gamedata folder for tgs2), and symlink parsed-logs to a folder accessable by the webserver
//	you also need the runtime condenser in the folder rc, with the binary named rc
//	it creates .gz files, you can abuse http-gzip-static and a few rewrite rules in nginx to make nginx serve them up as http-gzip compressed text files.


//exit;
if (php_sapi_name() != "cli")
	exit;

define("USLEEPBASE", 1); //change this to set the speed/niceness, used to pause between iterations of loops that do IO.

$stop = TRUE; //change to true to disable most functionality of the script, except the compression of the src logs.

$overwrite = FALSE;
$configonly = FALSE;
$testcompressed = FALSE; 
$bypasscronstop = FALSE; 
$options = getopt("doc", array("diff", "overwrite", "configonly", "testcompressed", "bypasscronstop"));

if (isset($options["diff"]) || isset($options["d"])) {
	$stop = FALSE;
	define("USLEEPBASE", 15);
}
if (isset($options["overwrite"]) || isset($options["o"]))
	$overwrite = TRUE;
if (isset($options["configonly"]) || isset($options["c"]))
	$configonly = TRUE;
if (isset($options["testcompressed"]))
	$testcompressed = TRUE;
if (isset($options["bypasscronstop"]))
	$bypasscronstop = TRUE;

if (!$bypasscronstop) {
	//exit; //uncomment this to disable crontabs while allowing cli calls with --bypasscronstop
}

/*
for ($argv as $argn=>$arg)
	if ($argn > 0)
		switch ($arg) {
			case "-d":
			case "--diff":
				$stop = FALSE;
				break;
			case "-o":
			case "--overwrite":
				$overwrite = TRUE;
				break;
		}
if ($argc >= 2 && $argv[1] === "-diff")
	$stop = FALSE;
*/
proc_nice(19);
function getfoldersinfolder($folder) {
	$results = scandir($folder);
	$folders = array();
	foreach ($results as $result) {
		if (!is_dir($folder . '/' . $result)) 
			continue;
		if ($result === '.' or $result === '..')
			continue;

		$folders[] = $folder . '/' . $result;
	}
	return $folders;
}

function getfilesinfolder($folder) {
	$results = scandir($folder);
	$files = array();
	foreach ($results as $result) {
		if (!is_file($folder . '/' . $result)) 
			continue;
		if ($result === '.' or $result === '..')
			continue;

		$files[] = $folder . '/' . $result;
	}
	return $files;
}

function getroundsbyserver($server) {
	$years = getfoldersinfolder('server-gamedata/'.$server.'/data/logs');
	
	$logfiles = array();
	foreach ($years as $year) {

		if (!is_numeric(basename($year)))
			continue;
		
		$months = getfoldersinfolder($year);

		foreach ($months as $month) {
			if (!is_numeric(basename($month)))
				continue;
			$days = getfoldersinfolder($month);
			
			foreach ($days as $day) {
				$rounds = getfoldersinfolder($day);
				foreach ($rounds as $round) {
					
					$roundid = (int) substr(basename($round), 6);
					//echo "$round => $roundid\n";
					if ($roundid < 2)
						continue;
					
					$logfiles[] = $round;
				}
			}
		}
	}
	return $logfiles;
}
function compressfile($file, $target = null) { //this likely has a shell exploit since unix file names can contain single quotes
	if ($target) {
		if (is_compressed($file)) {
			echo "copy-compressing file:$file to $target\n";
			copy($file, $target.'.gz');
		} else {
			echo "compressing file:$file to $target\n";
			exec('gzip -kcf9 "'.$file.'" > "'.$target.'.gz"');
		}
		
	} else {
		if (!is_compressed($file)) {
			echo "compressing file:$file\n";
			exec('gzip -f9 "'.$file.'"');
		} else {
			echo "noop-compressing file:$file\n";
		}
		
	}
	usleep(USLEEPBASE);
}

function condense_runtimes($infile, $outfile) {
	echo "Condensing runtime: $infile -> $outfile\n";
	chdir('rc');
	exec('gunzip -kc ../"'.$infile.'" | ./rc -s | gzip -c9 > ../"'.$outfile.'"');
	chdir('..');
	usleep(USLEEPBASE);
}

function parseruntime($runtime, $newpath, $monthruntimes, $dayruntimes) {
	echo "parsing runtime: $runtime -> $newpath/runtime.log\n";

	$file = gzopen($runtime, 'rb');
	if ($file === false)
		return;
	$newfile = gzopen($newpath.'/runtime.txt.gz', 'wb9');
	if ($newfile === false) {
		gzclose($file);
		return;
	}

	//parse each line
	while (($line = gzgets($file)) !== false) {
		//Remove ips
		$line = preg_replace('/(?:(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])\.){3}(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]?|[0-9])/', '-censored-', $line);
		//Remove byond printed strings
		$line = preg_replace('/.*Cannot read \".*/', '-censored (string output)', $line);

		//write it to the public locations
		gzwrite($newfile, $line);
		if ($monthruntimes)
			gzwrite($monthruntimes, $line);
		if ($dayruntimes)
			gzwrite($dayruntimes, $line);
	}
	gzclose($file);
	gzclose($newfile);
	condense_runtimes($newpath.'/runtime.txt.gz', $newpath.'/runtime.condensed.txt.gz');

	//compressfile($newfilename.'-condensed.txt');
	echo "Parse finished on runtime: $runtime\n";
	usleep(USLEEPBASE);
}

function updateconfig($server) {
	$serverfiles = array('config/maps.txt', 'config/server_overrides.txt', 'data/paintings.json', 'data/photo_albums.json', 'data/photo_frames.json', 'data/custom_outfits.json');
	$serverfolders = array('data/minimaps', 'data/npc_saves', 'data/engravings');
	$diffedfolders = array('data/Diagnostics/Resources');
	$filteredfiles = array('config.txt', 'game_options.txt', 'mrp_shared_overrides.txt', 'server_overrides.txt', 'resources.txt');
	$sharedfiles = array('config.txt', 'game_options.txt', 'mrp_shared_overrides.txt', 'admins.txt', 'admin_nicknames.json', 'admin_nicknames.txt', 'admin_ranks.txt', 'awaymissionconfig.txt', 'dynamic.json', 'lavaruinblacklist.txt', 'motd.txt', 'mrp_motd.txt', 'policy.json', 'resources.txt', 'sillytips.txt', 'spaceruinblacklist.txt', 'tips.txt', 'unbuyableshuttles.txt', 'server_side_modifications.dm' => 'server_side_modifications.dm', 'dynamic.mrp.json');
	$sharedfolders = array('title_screens', 'title_screens/images');
	
	if (!file_exists('parsed-logs/'.$server))
			mkdir('parsed-logs/'.$server, 0775, true);
	foreach ($serverfolders as $serverfolder) { 
		if (!file_exists('server-gamedata/'.$server.'/'.$serverfolder))
			continue;
		foreach (getfilesinfolder('server-gamedata/'.$server.'/'.$serverfolder) as $file) {
			$target = 'parsed-logs/'.$server.'/'.$serverfolder.'/'.basename($file);
			if (!file_exists(dirname($target)))
				mkdir(dirname($target), 0775, true);
			
			//@copy($file, $target);
			if (in_array(basename($file), $filteredfiles)) {
				filterconfig($file, $target);
				continue;
			}
			@compressfile($file, $target);
		}
	}
	
	foreach ($diffedfolders as $difffolder) {
		if (!file_exists('server-gamedata/'.$server.'/'.$difffolder))
			continue;
		$stop = false;
		foreach (array_reverse(getfilesinfolder('server-gamedata/'.$server.'/'.$difffolder)) as $file) {
			if (!ctype_digit(substr(basename($file), 0, 1)))
				continue;
			$target = 'parsed-logs/'.$server.'/'.$difffolder.'/'.basename($file);
			if (!file_exists(dirname($target)))
				mkdir(dirname($target), 0775, true);
			echo "$file -> $target\n";
			if (file_exists($target.'.gz'))
				$stop = true;
			//@copy($file, $target);
			@compressfile($file, $target);
			if ($stop)
				break;
		}
	}
	
	foreach ($sharedfiles as $source => $target) {
		if (is_numeric($source)) {
			$source = $target;
			$target = 'config/'.$target;
		}
		if (!file_exists('server-gamedata/shared/'.$source))
			continue;
		$target = 'parsed-logs/'.$server.'/'.$target;
		if (!file_exists(dirname($target)))
			mkdir(dirname($target), 0775, true);
		if (in_array(basename($source), $filteredfiles)) {
			filterconfig('server-gamedata/shared/'.$source, $target);
			continue;
		}
		compressfile('server-gamedata/shared/'.$source, $target);
		//compressfile('parsed-logs/'.$server.'/config/'.$serverfile);
	}
	
	foreach ($sharedfolders as $sharedfolder) { 
		if (!file_exists('server-gamedata/shared/'.$sharedfolder))
			continue;
		foreach (getfilesinfolder('server-gamedata/shared/'.$sharedfolder) as $file) {
			$target = 'parsed-logs/'.$server.'/config/'.$sharedfolder.'/'.basename($file);
			
			if (!file_exists(dirname($target)))
				mkdir(dirname($target), 0775, true);
			
			//@copy($file, $target);
			if (in_array(basename($file), $filteredfiles)) {
				filterconfig($file, $target);
				continue;
			}
			@compressfile($file, $target);
		}
	}
	
	foreach ($serverfiles as $serverfile) {
		if (!file_exists('server-gamedata/'.$server.'/'.$serverfile))
			continue;
		$target = 'parsed-logs/'.$server.'/'.$serverfile;
		if (!file_exists(dirname($target)))
			mkdir(dirname($target), 0775, true);
		if (in_array(basename($serverfile), $filteredfiles)) {
			filterconfig('server-gamedata/'.$server.'/'.$serverfile, $target);
			continue;
		}
		compressfile('server-gamedata/'.$server.'/'.$serverfile, $target);
		//compressfile('parsed-logs/'.$server.'/config/'.$serverfile);
	}
	
	/*foreach ($filteredfiles as $serverfile) {
		filterconfig($server, $serverfile);
	}*/
	
	//Paintings
	$painting_categories = getfoldersinfolder('server-gamedata/'.$server.'/data/paintings');
	foreach ($painting_categories as $category) {
		$target_dir = 'parsed-logs/'.$server.'/data/paintings/'.basename($category);
		if (!file_exists($target_dir))
			mkdir($target_dir, 0775, true);
		foreach (getfilesinfolder($category) as $file) {
			$target = $target_dir.'/'.basename($file);
			if (!file_exists($target.'.gz'))
				@compressfile($file, $target);
		}
	}
		
}
function filterconfig($configfile, $target) {
	echo "filtering $configfile\n";
	$filteredconfigs = array('COMMS_KEY', 'MEDAL_HUB_PASSWORD', 'DISCORD_TOKEN', 'ADMINHELP_WEBHOOK_URL', 'URGENT_ADMINHELP_WEBHOOK_URL', 'REGULAR_ADMINHELP_WEBHOOK_URL');
	$file = fopen($configfile, 'rb');
	if ($file === false)
		return;
	$newfile = gzopen($target.'.gz', 'wb9');
	if ($newfile === false) {
		fclose($file);
		return;
	}
	while (($line = fgets($file)) !== false) {
		foreach ($filteredconfigs as $filter)
			if (strstr(strtoupper($line), strtoupper($filter)) !== false)
				$line = '#'.$filter.' -FILTERED-'."\n";

		gzwrite($newfile, $line);
	}
	gzclose($newfile);
	echo "done filtering $configfile\n";
	usleep(USLEEPBASE);
}

function fillzips($file, $basename, $monthzip, $dayzip, $roundzip, $day, $round) {
	return;
	$handle = gzopen($file, 'r');
	
	$monthzip[$day.'/'.$round.'/'.$basename] = $handle;
	$monthzip[$day.'/'.$round.'/'.$basename]->compress(Phar::BZ2);
	
	gzrewind($handle);
	
	$dayzip[$round.'/'.$basename] = $handle;
	$dayzip[$round.'/'.$basename]->compress(Phar::BZ2);
	
	gzrewind($handle);
	
	$roundzip[$basename] = $handle;
	$roundzip[$basename]->compress(Phar::BZ2);

	gzclose($handle);
	usleep(USLEEPBASE);
}

function fill_round_zip($file, $basename, &$roundzip, $day, $round) {
	if (!($roundzip instanceof PharData)) {
		$roundzip = new PharData($roundzip, Phar::CURRENT_AS_FILEINFO | Phar::KEY_AS_FILENAME, null, Phar::ZIP);
		$roundzip->startBuffering();
	}
	$handle = gzopen($file, 'r');
	
	$roundzip[$basename] = $handle;
	$roundzip[$basename]->compress(Phar::BZ2);

	gzclose($handle);
	usleep(USLEEPBASE);
}

function is_compressed($file) {
	if (substr(basename($file), -3) == '.gz') {
		return true;
	}
	return false;
}

echo "Starting up\n";
$servers = array('terry', 'manuel', 'sybil', 'basil', 'event-hall-us', 'event-hall');
//$servers = array('terry');
foreach ($servers as $server) {
	echo "Loading $server\n";
	//if ($testcompressed && $server == 'basil')
		//break;
	if (!$testcompressed) {
		updateconfig($server);
		copy_pictures($server);
	}
	if ($configonly)
		continue;
	
	//parseruntimes($server);
	//continue;
	echo "Parsing logs files\n";
	$basenewpath = 'parsed-logs/'.$server.'/data/logs';
	$first = true;
	$last = false;
	$years = array_reverse(getfoldersinfolder('server-gamedata/'.$server.'/data/logs'));
	$dozips = false;
	
	foreach ($years as $year) {
		$baseyear = basename($year);
		if (!is_numeric($baseyear))
			continue;
		
		$months = array_reverse(getfoldersinfolder($year));
		foreach ($months as $month) {
			$basemonth = basename($month);
			$monthruntimes = null;
			$wrotemonth = false;
			if (!is_numeric($basemonth))
				continue;
			if (!file_exists("$basenewpath/$baseyear/$basemonth"))
				mkdir("$basenewpath/$baseyear/$basemonth", 0775, true);
			if ($dozips) {
				$monthzip = new PharData("$basenewpath/$baseyear/month.$baseyear-$basemonth.zip", Phar::CURRENT_AS_FILEINFO | Phar::KEY_AS_FILENAME, null, Phar::ZIP);
				$monthzip->startBuffering();
			}
			if ($stop)
				$monthruntimes = null /*gzopen("$basenewpath/$baseyear/$basemonth/$baseyear-$basemonth.runtime.txt.gz", "ab9")*/;
			$days = array_reverse(getfoldersinfolder($month));
			foreach ($days as $day) {
				$baseday = basename($day);
				$dayruntimes = null;
				$wroteday = false;
				if (!file_exists("$basenewpath/$baseyear/$basemonth/$baseday"))
					mkdir("$basenewpath/$baseyear/$basemonth/$baseday", 0775, true);
				if ($dozips) {
					$dayzip = new PharData("$basenewpath/$baseyear/$basemonth/day.$baseyear-$basemonth-$baseday.zip", Phar::CURRENT_AS_FILEINFO | Phar::KEY_AS_FILENAME, null, Phar::ZIP);
					$dayzip->startBuffering();
				}
				if ($stop)
					$dayruntimes = gzopen("$basenewpath/$baseyear/$basemonth/$baseday/$baseyear-$basemonth-$baseday.runtime.txt.gz", "ab9");
				$rounds = array_reverse(getfoldersinfolder($day));
				foreach ($rounds as $round) {
					$baseround = basename($round);
					$roundid = (int) substr($baseround, 6);
					//echo "$round => $roundid\n";
					if ($roundid < 2)
						continue;

					if ($first) {//skip first as its the current round
						$first = false;
						continue;
					}

					$roundA = explode('/', $round);
					$roundA[0] = 'parsed-logs';
					$newpath = implode('/', $roundA);
					
					if (!file_exists($newpath)) {
						mkdir($newpath,0775,true);
						
					} else if ($stop && !($testcompressed)) {
						if ($dozips) {
							$dayzip->stopBuffering();
							$monthzip->stopBuffering();
						}
						//gzclose($monthruntimes);
						//condense_runtimes("$basenewpath/$baseyear/$basemonth/$baseyear-$basemonth.runtime.txt.gz", "$basenewpath/$baseyear/$basemonth/$baseyear-$basemonth.runtime.condensed.txt.gz");
						gzclose($dayruntimes);
						condense_runtimes("$basenewpath/$baseyear/$basemonth/$baseday/$baseyear-$basemonth-$baseday.runtime.txt.gz", "$basenewpath/$baseyear/$basemonth/$baseday/$baseyear-$basemonth-$baseday.runtime.condensed.txt.gz");

						break 4;
					}
					$roundzip = $newpath.'.zip';
					
					$logfiles = getfilesinfolder($round);
					foreach ($logfiles as $logfile) {
						$basename = basename($logfile);
						if (!is_compressed($logfile)) {
							//echo "$logfile is not compressed\n";
							compressfile($logfile);
							if (file_exists($logfile.'.gz'))
								$logfile .= '.gz';
						} else {
							//echo "$logfile is compressed\n";
							$basename = basename($logfile, '.gz');
						}
						if ($testcompressed)
							$basename = "";
						continue; //remove here mso
						switch ($basename) {
							case 'game.log':
								$basefilename = basename($basename, '.log');
								$fullnewpath = $newpath.'/'.$basefilename.'.txt';
								
								if ($overwrite || !file_exists($fullnewpath.'.gz')) {
									parselog($logfile, $newpath, TRUE, TRUE);
									if ($dozips)
										fillzips($fullnewpath.'.gz', $basefilename.'.txt', $monthzip, $dayzip, $roundzip, $baseday, $baseround);
									else
										fill_round_zip($fullnewpath.'.gz', $basefilename.'.txt', $roundzip, $baseday, $baseround);
									
									$fullnewpath = $newpath.'/'.$basefilename.'.html';
									if ($dozips)
										fillzips($fullnewpath.'.gz', $basefilename.'.html', $monthzip, $dayzip, $roundzip, $baseday, $baseround);
									else
										fill_round_zip($fullnewpath.'.gz', $basefilename.'.html', $roundzip, $baseday, $baseround);
								}
								break;
							
							case 'runtime.log':
								$basefilename = basename($basename, '.log');
								$fullnewpath = $newpath.'/'.$basefilename.'.txt';
								
								if ($overwrite || !file_exists($fullnewpath.'.gz')) {
									parseruntime($logfile, $newpath, $monthruntimes, $dayruntimes);
									
									if ($dozips)
										fillzips($fullnewpath.'.gz', $basefilename.'.txt', $monthzip, $dayzip, $roundzip, $baseday, $baseround);
									else
										fill_round_zip($fullnewpath.'.gz', $basefilename.'.txt', $roundzip, $baseday, $baseround);
									
									$fullnewpath = $newpath.'/'.$basefilename.'.condensed.txt';
									
									if ($dozips)
										fillzips($fullnewpath.'.gz', $basefilename.'.condensed.txt', $monthzip, $dayzip, $roundzip, $baseday, $baseround);
									else
										fill_round_zip($fullnewpath.'.gz', $basefilename.'.condensed.txt', $roundzip, $baseday, $baseround);
									
								}
								break;
							
							case 'sql.log':
								$basefilename = basename($basename, '.log');
								$fullnewpath = $newpath.'/'.$basefilename.'.txt';
								
								if ($overwrite || !file_exists($fullnewpath.'.gz')) {								
									parselog($logfile, $newpath, FALSE, FALSE);
									
									if ($dozips)
										fillzips($fullnewpath.'.gz', $basefilename.'.txt', $monthzip, $dayzip, $roundzip, $baseday, $baseround);
									else
										fill_round_zip($fullnewpath.'.gz', $basefilename.'.txt', $roundzip, $baseday, $baseround);
									
									
								}
								break;
							case 'attack.log':
							case 'dynamic.log':
							case 'qdel.log':
							case 'initialize.log':
							case 'pda.log':
							case 'telecomms.log':
							case 'overlay.log':
							case 'manifest.log':
							case 'job_debug.log':
							case 'virus.log':
							case 'econ.log':
							case 'signals.log':
							case 'shuttle.log':
							case 'mecha.log':
							case 'asset.log':
							case 'map_errors.log':
							case 'cloning.log':
							case 'uplink.log':
							case 'paper.log':
							case 'harddel.log':
							case 'silicon.log':
							case 'tools.log':
							case 'mob_tags.log':
							case 'harddels.log':
							case 'speech_indicators.log':
							#case 'debug.log':
							case 'economy.log':
							#case 'filter.log':
							case 'tool.log':
							case 'signal.log':
							case 'silo.log':
							case 'tool.log':
								$basefilename = basename($basename, '.log');
								$fullnewpath = $newpath.'/'.$basefilename.'.txt';
								
								if ($overwrite || !file_exists($fullnewpath.'.gz')) {
									compressfile($logfile, $fullnewpath);
									
									if ($dozips)
										fillzips($fullnewpath.'.gz', $basefilename.'.txt', $monthzip, $dayzip, $roundzip, $baseday, $baseround);
									else
										fill_round_zip($fullnewpath.'.gz', $basefilename.'.txt', $roundzip, $baseday, $baseround);
										
									
								}
								break;
							
							case 'attack.log.json':
							case 'dynamic.log.json':
							case 'qdel.log.json':
							case 'initialize.log.json':
							case 'pda.log.json':
							case 'telecomms.log.json':
							case 'overlay.log.json':
							case 'manifest.log.json':
							case 'job_debug.log.json':
							case 'virus.log.json':
							case 'econ.log.json':
							case 'signals.log.json':
							case 'shuttle.log.json':
							case 'mecha.log.json':
							case 'asset.log.json':
							case 'map_errors.log.json':
							case 'cloning.log.json':
							case 'uplink.log.json':
							case 'paper.log.json':
							case 'harddel.log.json':
							case 'silicon.log.json':
							case 'tools.log.json':
							case 'mob_tags.log.json':
							case 'harddels.log.json':
							case 'speech_indicators.log.json':
							#case 'debug.log.json':
							case 'economy.log.json':
							#case 'filter.log.json':
							case 'tool.log.json':
							case 'signal.log.json':
							case 'silo.log.json':
							case 'tool.log.json':
							
							case 'kudzu.html':
							case 'wires.html':
							case 'atmos.html':
							case 'cargo.html':
							case 'deaths.html':
							case 'gravity.html':
							case 'records.html':
							case 'singulo.html':
							case 'experimentor.html':
							case 'supermatter.html':
							case 'engine.html':
							case 'botany.html':
							case 'presents.html':
							case 'telesci.html':
							case 'research.html':
							case 'radiation.html':
							case 'portals.html':
							case 'hallucinations.html':
							case 'hypertorus.html':
							case 'circuit.html':	
							case 'nanites.html':
							case 'newscaster.json':
							case 'dynamic.json':
							case 'round_end_data.json':
							case 'round_end_data.html':
							case 'profiler.json':
							case 'sendmaps.json':
							case 'id_card_changes.html':
							case 'target_zone_switch.json':
							case 'silo.json':
							case 'crafting.html':
							case 'init_profiler.json':
							case 'init_times.json':
							case ((substr($basename, 0, 5) == 'perf-') ? $basename : !$basename):
								$fullnewpath = $newpath.'/'.$basename;
								
								if ($overwrite || !file_exists($fullnewpath.'.gz')) {
									compressfile($logfile, $fullnewpath);
									
									if ($dozips)
										fillzips($fullnewpath.'.gz', $basename, $monthzip, $dayzip, $roundzip, $baseday, $baseround);
									else
										fill_round_zip($fullnewpath.'.gz', $basename, $roundzip, $baseday, $baseround);
									
								}
								break;
							case 'config_error.log':
							case 'hrefs.html':
							case 'hrefs.log':
							case 'suspicious_logins.log':
							case 'tgui.log':
							case 'dd.log':
							case 'filters.log':
								break;
							default:
								if (!empty($basename))
									echo "!!(default) $basename => $logfile\n";
								break;
						}
						
						usleep(USLEEPBASE);
					}
					$logfolders = getfoldersinfolder($round);
					foreach ($logfolders as $logfolder) {
						$basename = basename($logfolder);
						switch ($basename) {
							case 'photos':
								if (!file_exists($newpath.'/photos'))
										mkdir($newpath.'/photos',0775,true);
								foreach (getfilesinfolder($logfolder) as $picturefile) {
									$basename = basename($picturefile);
									$fullnewpath = $newpath.'/photos/'.$basename;
									
									copy($picturefile, $fullnewpath);
									
									if ($dozips)
										fillzips($fullnewpath, 'photos/'.$basename, $monthzip, $dayzip, $roundzip, $baseday, $baseround);
									else
										fill_round_zip($fullnewpath, 'photos/'.$basename, $roundzip, $baseday, $baseround);
								}
							break;
							case 'profiler':
								if (!file_exists($newpath.'/profiler'))
										mkdir($newpath.'/profiler',0775,true);
								foreach (getfilesinfolder($logfolder) as $logfile) {
									$basename = basename($logfile);
									$fullnewpath = $newpath.'/profiler/'.$basename;
									if (!is_compressed($logfile)) {
										compressfile($logfile);
										if (file_exists($logfile.'.gz'))
											$logfile .= '.gz';
									} else {
										$basename = basename($logfile, '.gz');
									}
									continue; //here mso
									compressfile($logfile, $fullnewpath);
									
									if ($dozips)
										fillzips($fullnewpath, 'profiler/'.$basename, $monthzip, $dayzip, $roundzip, $baseday, $baseround);
									else
										fill_round_zip($fullnewpath, 'profiler/'.$basename, $roundzip, $baseday, $baseround);
								}
							break;
							case 'secret':
								foreach (getfilesinfolder($logfolder) as $logfile) {
									$basename = basename($logfile);
									$fullnewpath = $newpath.'/profiler/'.$basename;
									if (!is_compressed($logfile)) {
										compressfile($logfile);
										if (file_exists($logfile.'.gz'))
											$logfile .= '.gz';
									} else {
										$basename = basename($logfile, '.gz');
									}
								}
							break;
							default:
								echo "(folder default) $logfile => $newpath/$basename\n";
								break;
						}
						usleep(USLEEPBASE);
						
					}
					if ($roundzip instanceof PharData)
						$roundzip->stopBuffering();
				}
				if ($dozips)
					$dayzip->stopBuffering();
				if ($stop) {
					gzclose($dayruntimes);
					condense_runtimes("$basenewpath/$baseyear/$basemonth/$baseday/$baseyear-$basemonth-$baseday.runtime.txt.gz", "$basenewpath/$baseyear/$basemonth/$baseday/$baseyear-$basemonth-$baseday.runtime.condensed.txt.gz");
				}
				usleep(USLEEPBASE);
			}
			if ($dozips)
				$monthzip->stopBuffering();
			if ($stop) {
				//gzclose($monthruntimes);
				//condense_runtimes("$basenewpath/$baseyear/$basemonth/$baseyear-$basemonth.runtime.txt.gz", "$basenewpath/$baseyear/$basemonth/$baseyear-$basemonth.runtime.condensed.txt.gz");
			}
			usleep(USLEEPBASE);
		}
		usleep(USLEEPBASE);
	}
	usleep(USLEEPBASE);
}

function parselog($logfile, $newpath, $lineparse, $htmlify) {
	//echo "Parsing logfile: $logfile\n";

	$filename = basename($logfile, '.gz');
	
	echo "Parsing logfile as Game Log: $logfile\n";
	$file = gzopen($logfile, "rb");
	if ($file === false) {
		echo "file === false\n";
		return;
	}
	$tofile = gzopen($newpath.'/'.basename($filename, '.log').'.txt.gz', "wb9");
	if ($tofile === false) {
		echo "tofile === false\n";
		@gzclose($file);
		return;
	}
	if (!is_resource($file) || !is_resource($tofile)) {
		echo "is_resource === false\n";
		@gzclose($file);
		@gzclose($tofile);
		return;
	}
	
	$tofilehtml = NULL;
	if ($htmlify) {
		$tofilehtml = gzopen($newpath.'/game.html.gz', "wb9");
		if ($tofilehtml === false || !is_resource($tofilehtml)) {
			echo "tofilehtml === false\n";
			@gzclose($file);
			@gzclose($tofile);
			return;
		}
		gzwrite($tofilehtml, '<html><head><title>Log file: '.$newpath.' - /tg/station 13</title><link rel="stylesheet" type="text/css" href="/logfilestyle.css"></head><body>');
	}
	echo "checks passed\n";
	while (($line = gzgets($file)) !== false) {
		$rawline = trim($line, "\n\r");
		$parsedline = $rawline;
		if ($lineparse)
			$parsedline = parseline($rawline, htmlspecialchars($line));
		$html = "";
		if ($htmlify)
			$html = preg_replace('/(?:(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])\.){3}(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]?|[0-9])/', '<span class="censored">-censored(ip)-</span>', $parsedline[1]);
		
		$line = preg_replace('/(?:(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])\.){3}(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]?|[0-9])/', '-censored(ip)-', $parsedline[0]); //remove ips

		gzwrite($tofile, $line."\r\n");
		if ($htmlify)
			gzwrite($tofilehtml, $html);
	}
	if ($htmlify) {
		gzwrite($tofilehtml, '</body></html>');
		gzclose($tofilehtml);
	}
	gzclose($file);
	gzclose($tofile);
	echo "done parsing game log\n";
	usleep(USLEEPBASE);
}
	
	
function parseline ($line, $html) {
	$line = trim($line);
	if (!$line)
		return array('-censored(empty_line)-','<p class="censored">-censored(empty_line)-</p>');
	
	if ($line[0] != '[')
		return array('-censored(no_ts_start)-','<p class="censored">-censored(no_ts_start)-</p>');
	
	$log_timestamp = substr($line, 1, strpos($line, ']')-1);
	if (strlen($log_timestamp) > 23 || strlen($log_timestamp) < 8)
		return array('-censored(wrong_ts_size)-','<p class="censored">-censored(wrong_ts_size)-</p>');
	
	if (substr_count($log_timestamp, ':') != 2)
		return array('-censored(no_time_sep)-','<p class="censored">-censored(no_time_sep)-</p>');
	
	if (preg_match('/^([0-9]{2}:[0-9]{2}:[0-9]{2}|[0-9]{2,4}-[0-9]{2,4}-[0-9]{2,4} [0-9]{2}:[0-9]{2}:[0-9]{2}(\\.[0-9]{1,3})+)$/', $log_timestamp) === false)
		return array('-censored(no_ts_regex_match)-','<p class="censored">-censored(no_ts_regex_match)-</p>');
		
	$words = explode(' ', $line);
	$htmlwords = explode(' ', $html);
	
	$logtype = explode(']',$words[0]);
	
	if (count($logtype) < 2) {
		$logtype = $words[2];
		$words[0] .= ' '.$words[1].' '.$words[2];
		$htmlwords[0] .= ' '.$htmlwords[1].' '.$htmlwords[2];
		array_splice($words, 1, 2);
		//echo "$words[0]|||$words[1]|||$words[2]|||$logtype\n";
	} else {
		$logtype = $logtype[1];
	}
	if ($logtype == "GAME-COMPAT:") {
		$logtype = $words[1];
		$words[0] .= ' '.$logtype;
		array_splice($words, 1, 1);
		
	}


	$logsubtypes = explode("-", $logtype);
	if (count($logsubtypes) > 1) {
		if ($logsubtypes[0] == "GAME") {
			array_splice($logsubtypes, 0, 1);
		}
		$logtype = $logsubtypes[0];
		if (count($logsubtypes) > 1) {
			array_splice($words, 1, 0, array_slice($logsubtypes, 1));
		}
	}
	
	if (substr($logtype, -1) != ':')
		return censor('no_category_colon');
	
	if ($logtype !== strtoupper($logtype))
		return censor('first_word_not_uppercase');
	
	switch ($logtype) {
		case 'ACCESS:':
			if ($words[1] == 'Login:') {
				$words[count($words)-4] = '-censored(ip/cid)-';
				$htmlwords[count($htmlwords)-4] = '<span class="censored">-censored(ip/cid)-</span>';
			}
			if ($words[1] == 'Failed')
				return censor('invalid connection data');
			break;
		case 'ADMIN:':
			if ($words[1] == 'HELP:')
				return censor('asay/apm/ahelp/notes/etc');
			if ($words[1] == 'PM:')
				return censor('asay/apm/ahelp/notes/etc');
			if ($words[1] == 'ASAY:')
				return censor('asay/apm/ahelp/notes/etc');
			if (preg_match('/ADMIN: .*\\/\\(.*\\) : /', $line))
				return censor('asay/apm/ahelp/notes/etc');
			if (preg_match('/ADMIN: .*\\/\\(.*\\) added note /', $line))
				return censor('asay/apm/ahelp/notes/etc');
			if (preg_match('/ADMIN: .*\\/\\(.*\\) removed a note /', $line))
				return censor('asay/apm/ahelp/notes/etc');
			if (preg_match('/ADMIN: .*\\/\\(.*\\) has added /', $line))
				return censor('asay/apm/ahelp/notes/etc');
			if (preg_match('/ADMIN: .*\\/\\(.*\\) has edited /', $line))
				return censor('asay/apm/ahelp/notes/etc');
			if (preg_match('/ADMIN: [^:]*\\/\\(.*\\) ".*"/', $line))
				return censor('asay/apm/ahelp');
			/*if (preg_match('/ADMIN: .*\\/\\(.*\\) has created a /', $line))
				return censor('asay/apm/ahelp/notes/etc');
			if (preg_match('/ADMIN: .*\\/\\(.*\\) has removed a /', $line))
				return censor('asay/apm/ahelp/notes/etc');
			if (preg_match('/ADMIN: .*\\/\\(.*\\) has deleted a /', $line))
				return censor('asay/apm/ahelp/notes/etc');*/ //old messages about message/note/memo adding.
			if ($words[1] == '<a')
				return censor('asay/apm/ahelp/notes/etc');
			break;
		case 'ADMINPRIVATE:':
			return censor('private logtype');
			break;
		case 'ASAY:':
			return censor('asay/apm/ahelp/notes/etc');
			break;
		case 'SQL:':
			return censor('sql logs');
			break;
		case 'TOPIC:':
			return censor('world_topic logs');
			break;
		case 'SAY:':
		case 'WHISPER:':
		case 'OOC:':
			foreach ($htmlwords as $i=>$word) {
				if ($word == ':') {
					$htmlwords[1] = '<b>'.$htmlwords[1];
					$htmlwords[$i-1] = $htmlwords[$i-1].'</b>';
					break;
				}
			}
			break;
		default:
			break;
	}		
	
	return array(implode(' ',$words), '<p class="'.rtrim(explode(']',$words[0])[1],':').'">'.implode(' ',$htmlwords).'</p>');
}

function censor($reason) {
	return array('-censored('.$reason.')-','<p class="censored">-censored('.$reason.')-</p>');
}

function copy_pictures($server) {
	global $stop, $overwrite;
	echo "Parsing picture log files\n";
	$basenewpath = 'parsed-logs/'.$server.'/data/picture_logs';
	$first = true;
	$last = false;
	$years = array_reverse(getfoldersinfolder('server-gamedata/'.$server.'/data/picture_logs'));
	$dozips = false;
	
	foreach ($years as $year) {
		$baseyear = basename($year);
		if (!is_numeric($baseyear))
			continue;
		
		$months = array_reverse(getfoldersinfolder($year));
		foreach ($months as $month) {
			$basemonth = basename($month);
			if (!is_numeric($basemonth))
				continue;
			if (!file_exists("$basenewpath/$baseyear/$basemonth"))
				mkdir("$basenewpath/$baseyear/$basemonth", 0775, true);
			
			$days = array_reverse(getfoldersinfolder($month));
			foreach ($days as $day) {
				$baseday = basename($day);
				if (!file_exists("$basenewpath/$baseyear/$basemonth/$baseday"))
					mkdir("$basenewpath/$baseyear/$basemonth/$baseday", 0775, true);
				$rounds = array_reverse(getfoldersinfolder($day));
				foreach ($rounds as $round) {
					$baseround = basename($round);
					$roundid = (int) substr($baseround, 6);
					//echo "$round => $roundid\n";
					if ($roundid < 2)
						continue;

					if ($first) { //skip first as its the current round
						$first = false;
						continue;
					}

					$roundA = explode('/', $round);
					$roundA[0] = 'parsed-logs';
					$newpath = implode('/', $roundA);
					
					if (!file_exists($newpath)) {
						mkdir($newpath,0775,true);
					} else if ($stop) {
						break 4;
					}
					
					$pictures = getfilesinfolder($round);
					foreach ($pictures as $picture) {
						$basename = basename($picture);
						$fullnewpath = $newpath.'/'.$basename;
						if (!$overwrite && file_exists($fullnewpath))
							continue;
						copy($picture, $newpath.'/'.$basename);
					}
				}
			}
		}
	}
}

?>