Sunday, December 21, 2008

Programming: MAME roms cleaner

A ruby script for cleaning up a collection of MAME roms.
It essentially keeps the subset of roms necessary to play the games, and deletes the clones or non working games; it also separates some systems to ease management.
This may sound an offense to anally retentive people, which use to keep the roms for the sake of keeping them, or thinking that maybe in a thousand year a nonworking rom could be emulated correctly.

Notes:
* it's rule based, so one can actually write its own rules
* it only works in linux
* it caches (Marshal class) the data during the first run, as parsing the xml is darn slow
* it's written in ruby, so it requires the ruby interprer, plus the 'hpricot' gem
* it's redundant, for clarity


require 'rubygems'
require 'hpricot'
require 'fileutils'

################################################################################
# CLASS DEFINITIONS
################################################################################

class Game
PRELIMINARY = 'preliminary'
# GOOD = 'good'
# IMPERFECT = 'imperfect'

attr_reader :name, :sourcefile, :cloneof, :romof, :sampleof
attr_reader :description
attr_reader :status, :emulation, :color, :sound, :graphic
attr_accessor :parent, :sampleparent, :clones # clones not actually assigned, just modified

def year
(parent.year if parent) || @year || (sampleparent.year if sampleparent)
end

def has_chd?
@has_chd
end

def initialize(name, sourcefile, cloneof, romof, sampleof, description, year, status, emulation, color, sound, graphic, has_chd)
raise sprintf("%-8s: CO[%-8s] RO [%-8s]", name, cloneof, romof) if cloneof && (cloneof != romof)

@name, @sourcefile, @cloneof, @romof, @sampleof = name, sourcefile, cloneof, romof, sampleof
@description, @year = description, year
@status, @emulation, @color, @sound, @graphic = status, emulation, color, sound, graphic
@has_chd = has_chd
@parent, @sampleparent, @clones = nil, nil, []
end

def working?
@emulation != PRELIMINARY
end

def non_working?
! working?
end

def parent?
@cloneof.nil?
end

def clone?
! parent?
end

def self.decode(game_node)
driver_node = (game_node/'driver').first
description_node = (game_node/'description').first
year_node = (game_node/'year').first
has_chd = ((game_node/'disk')).size > 0

Game.new(game_node['name'], game_node['sourcefile'], game_node['cloneof'], game_node['romof'], game_node['sampleof'],
description_node.inner_html, (year_node.inner_html if year_node),
driver_node['status'], driver_node['emulation'], driver_node['color'], driver_node['sound'], driver_node['graphic'],
has_chd)
end
end

################################################################################
# ROUTINES
################################################################################

def create_dirs(roms_dirs)
puts '... creating dirs'
roms_dirs.each { |sym, dir| FileUtils.mkdir_p(dir)}
end

def load_games(mame_executable, cache_file)
if File.exists?(cache_file)
puts '... loading cached games'
open(cache_file, 'r') { |file| Marshal.load(file) }
else
puts '... piping mame xml games list into hpricot'
list_xml = IO.popen("#{mame_executable} -listxml") { |pipe| Hpricot::XML(pipe) }

games = parse_games(list_xml)

puts '... caching games'
open(cache_file, 'w') { |file| Marshal.dump(games, file) }

games
end
end

def load_xml(mame_executable)
end

def parse_games(list_xml)
games = {}

print '... decoding games'
(list_xml/'/mame/game').each do |xml_game|
game = Game.decode(xml_game)
games[game.name] = game
# print '.'
end
puts

print '... linking parents/clones'
games.each do |name, game|
if game.cloneof
parent = games[game.cloneof]
parent.clones << game if parent
game.parent = parent
end

if game.sampleof
sampleparent = games[game.sampleof]
game.sampleparent = sampleparent
end
# print '.'
end
puts

games
end

def filter_games(games, rules, roms_dirs)
puts '... filtering games'
games.each do |name, game|
if File.exist?(File.join(roms_dirs[:base], "#{name}.zip"))
filter = rules.call(game, roms_dirs)
puts filter if filter
end
end
end

################################################################################
# USER DEFINITIONS
################################################################################

MAME_EXECUTABLE = 'sdlmame'
MARSHALLED_FILE = '/tmp/list_mame_xml.rms'

ROMS_DIRS = {
:base => '/home/saverio/docs/roms/mame',
:neogeo => '/home/saverio/docs/roms/mame_neogeo',
:modern => '/home/saverio/docs/roms/mame_modern',
:chd => '/home/saverio/docs/roms/mame_chd'
}

# laser games aren't included, as currently there aren't the mame disk dumps, so
# it's better to use DAPHNE ones.
MODERN_SYSTEMS_DRIVERS = %w{
cps3.c # Capcom CPS3 => CPS3 Emulator
model1.c # Sega Model 1 => ???
model2.c # Sega Model 2 => Model 2 Emulator
namcos11.c # Namco System 11 => ZiNc
namcos12.c # Namco System 12 => ZiNc
naomi.c # Sega NAOMI => nullDC
}
#MODERN_OTHER = %w{
# stv.c # Sega Titan Video
# vegas.c # Atary/Midway Vegas
#}
#MODERN_UNEMULATED = %w{
# model3.c # Sega Model 3
# viper.c # Konami Viper
#}

# if a game is a preserved clone, don't delete it
# delete the very old games
# delete the nonworking clones, or the clones with a working parent
# keep/move the games better emulated by other emulators ('modern'), even if nonworking
# delete the parents of entirely nonworking gamesets
# move chd games
# other games are kept in the base dir
RULES_CLEAN = lambda do |game, roms_dirs|
filename = File.join(roms_dirs[:base], game.name) + "{,.zip}"

if game.cloneof == 'sf2ce'
# do nothing
elsif (! game.year) || game.year.to_i < 1980
sprintf "%-32s # Delete seventies game", "rm #{filename}"
elsif game.clone? && game.parent.working?
sprintf "%-32s # Delete clone of working parent", "rm #{filename}"
elsif game.clone? && game.non_working?
sprintf "%-32s # Delete nonworking clone", "rm #{filename}"
elsif game.sourcefile == 'neodrvr.c'
sprintf "%-32s # Move neo geo game", "mv #{filename} #{roms_dirs[:neogeo]}"
elsif MODERN_SYSTEMS_DRIVERS.include?(game.sourcefile)
sprintf "%-32s # Move modern game", "mv #{filename} #{roms_dirs[:modern]}"
elsif game.parent? && game.non_working? && ! game.clones.any? { |clone| clone.working? }
sprintf "%-32s # Delete parent of non working gameset", "rm #{filename}"
elsif game.has_chd?
sprintf "%-32s # Move chd game", "mv #{filename} #{roms_dirs[:chd]}"
end
end

# example, for displaying modern games with chd
RULES_MODERN_CHD = lambda do |game, roms_dirs|
puts game.name if game.has_chd?&& MODERN_SYSTEMS_DRIVERS.include?(game.sourcefile)
end

################################################################################
# RUNNER
################################################################################

create_dirs(ROMS_DIRS)
games = load_games(MAME_EXECUTABLE, MARSHALLED_FILE)

filter_games(games, RULES_CLEAN, ROMS_DIRS)

#filter_games(games, RULES_MODERN_CHD , :base => '/home/saverio/docs/roms/mame_modern')

Sunday, December 7, 2008

Programming: Lossless editing (cut/paste) of quicktime movies

Modern cameras use the quicktime format, which is very easy to cut/paste losslessly.
This means that one can edit the movies shot during the vacations without worsening the already (relatively) low quality.

Specifically, it takes one or more files with the timings (start/end) and merges them into a destination file.

It's not usable by a non-programmer, but actually it's very easy to create a gui for it; unfortunately java doesn't help deadly simple applications deploying.


/*
* License: you can do what the heck you want with this file, as long as you
* reference me as starting author.
*
* @author Saverio Miroddi (com.inbox@pub.saverio - reverse the tokens to get
* the email)
*/
import quicktime.QTException;
import quicktime.QTSession;
import quicktime.io.IOConstants;
import quicktime.io.OpenMovieFile;
import quicktime.io.QTFile;
import quicktime.std.StdQTConstants;
import quicktime.std.movies.Movie;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

class SourceData
{
public final String fileName;
public final float start;
/** -1 = end */
public final float end;

public SourceData(String fileName, float start, float end)
{
this.fileName = fileName;
this.start = start;
this.end = end;
}
}

/**
* Edits (cuts/pastes) pieces of quicktime movies into a destination quicktime
* file.

* The advantage of using this method is that it's lossless - very useful when
* editing movies shot with a portable camera, which are already (relatively)
* low quality.
*


* Requires quicktime wrapper (QTJava.zip), that is installed along with
* QuickTime, in the lib/ext folder of the JRE); at the time of writing, the QT
* version is 7.5
*


* Note: I never tried executing it from the commandline, I always edited the
* params directly in the code, but it should work fine.
*/
public class QuickTimeEditing
{
private static final int SOURCE_TIMESCALE = 30;
private static final String REF_FILENAME = "qt_edit.ref.tmp";

public static void main(String[] args) throws Exception
{
args = new String[] {
"d:/tmp/male_singer_falsetto.mov",
"d:/desktop/videos/heli0.mov", "0", "-1",
"d:/desktop/videos/heli1.mov", "0", "-1",
"d:/desktop/videos/heli8.mov", "0", "-1",
"d:/desktop/videos/heli9.mov", "0", "-1",
"d:/desktop/videos/heli10.mov", "0", "-1",
};

String destFile = args[0];
SourceData[] sourceData = extractSourceData(args);

new QuickTimeEditing().edit(destFile, sourceData);
}

/**
* @param args first is skipped
*/
private static SourceData[] extractSourceData(String... args)
{
List sourceData = new ArrayList();

for (int i = 1; i < args.length; )
{
String filename = args[i++];
float start = Float.parseFloat(args[i++]);
float end = Float.parseFloat(args[i++]);

sourceData.add(new SourceData(filename, start, end));
}

return sourceData.toArray(new SourceData[0]);
}

// PUBLIC INSTANCE METHODS /////////////////////////////////////////////////

public void edit(String destFile, SourceData... sourcesData) throws Exception
{
String refFile = createRefFilename(destFile);

try {
System.out.println("Opening...");

QTSession.open();

joinSourceMovies(destFile, refFile, sourcesData);
}
finally {
System.out.println("Closing...");

QTSession.close();

System.out.println("Cleaning up...");

cleanupRefFiles(refFile);
}
}

// PRIVATE INSTANCE METHODS /////////////////////////////////////////////////

private final String createRefFilename(String destFile) throws IOException
{
File destFileDir = new File(destFile).getParentFile();
return new File(destFileDir, REF_FILENAME).getCanonicalPath();
}

private void joinSourceMovies(String destFile, String refFile, SourceData... sourcesData) throws QTException, IOException
{
Movie refMovie = Movie.createMovieFile(new QTFile(refFile),
StdQTConstants.kMoviePlayer, StdQTConstants.createMovieFileDeleteCurFile);
refMovie.setTimeScale(SOURCE_TIMESCALE);

int lastTimeScaled = 0;

for (SourceData sourceData : sourcesData)
{
System.out.println("Processing '" + sourceData.fileName + "' (" + sourceData.start + "->" + sourceData.end + ")...");

Movie sourceMovie = Movie.fromFile(OpenMovieFile.asRead(new QTFile(sourceData.fileName)));
int timeScale = sourceMovie.getTimeScale();

if (timeScale != SOURCE_TIMESCALE) throw new RuntimeException("Too lazy to process time scales != " + SOURCE_TIMESCALE + ": " + timeScale);

int startTimeScaled = (int)(sourceData.start * timeScale);
int lengthScaled = (int)(sourceData.end > 0 ?
sourceData.end * timeScale :
sourceMovie.getDuration() - startTimeScaled);

sourceMovie.insertSegment(refMovie, startTimeScaled, lengthScaled, lastTimeScaled);

lastTimeScaled += lengthScaled;
}

refMovie.flatten (0, // movieFlattenFlags
new QTFile(destFile),
StdQTConstants.kMoviePlayer, // creator
IOConstants.smSystemScript, // scriptTag
StdQTConstants.createMovieFileDeleteCurFile, // createQTFileFlags
StdQTConstants.movieInDataForkResID, // resId
destFile);

}

private void cleanupRefFiles(String refFile)
{
new File(refFile).delete();
new File(refFile + ".#res").delete();
}
}