######################################## # The contents of this file are subject to the MLX PUBLIC LICENSE version # 1.0 (the "License"); you may not use this file except in # compliance with the License. # # Software distributed under the License is distributed on an "AS IS" # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See # the License for the specific language governing rights and limitations # under the License. # # The Original Source Code is "compClust", released 2003 September 03. # # The Original Source Code was developed by the California Institute of # Technology (Caltech). Portions created by Caltech are Copyright (C) # 2002-2003 California Institute of Technology. All Rights Reserved. ######################################## # # Filename : ANN.py # Description : Artificial Neural Network Wrapper # Author(s) : Ben Bornstein # Organization : Machine Learning Systems, Jet Propulsion Laboratory # Created : February 2002 # Revision : $Id: ANN.py,v 1.9 2004/04/02 02:40:14 diane Exp $ # Source : $Source: /proj/CVS/code/python/compClust/mlx/wrapper/ANN.py,v $ # """ Artificial Neural Network (ANN) Wraps the University of Wisconsin Backpropagation (UWBP) package in a generic and consistent API for supervised Machine Learning algorithms. UWBP is a sophisticated package for training Artificial Neural Networks (ANNs) via the Backpropagation algorithm. It was written in 1992 and updated periodically thereafter by Richard Maclin and David Opitz while under the academic advisement of Jude Shavlik of the UW-Madison Computer Science Department. The UWBP code is based on ideas from McClelland and Reumelhart [1]. See the UWBP README (imported/uwbp/README) for more information. ANN.py depends on the environment variable: ANN_COMMAND (usually in imported/uwbp/bpLINUX) The following parameters are tunable when training the ANN: seed Integer. Initializes a random number generator used to assign random values to network weights before training begins. Default = 42. lrate Real, range [0, 1]. Learning Rate sets the rate at which the network converges to (learns) a model. The value of learning rate tends to produce a tradeoff in the number of iterations required to converge to a solution versus the overall quality of the solution. Such quality is often measured in the ANN's ability to generalize to predict data not seen during training. In general, values near 0 tend to yield higher quality models, but require more iterations to converge, while values near 1 yield lower quality models, but require less iterations to converge. Default = .002. numIterations Integer, > 0. Each iteration is also referred to as an epoch. An epoch occurs after each pattern (datum) has been presented to the network and prediction errors have been backpropagated through the network to update network weights. The order in which patterns are presented is randomly rearranged after each epoch. Default = 10000. hiddenUnits Space separated list of integers, each > 0, may be empty. The network may contain zero or more layers of hidden units and each hidden layer may have one or more hidden units. Each successive integer in the list indicates the number of hidden units in that hidden layer. Default = '' (no hidden units). An example parameter set is: seed = '1234' lrate = '.012' numIterations = '1000' hiddenUnits = '2 2' References: [1] James L. McClelland and David E. Rumelhart. (1988). 'Explorations in Parallel and Distributed Processing', MIT Press: Cambridge, MA. """ import os import string import tempfile import Numeric import compClust.mlx.wrapper import compClust.util from compClust.mlx.Supervised import Supervised from compClust.mlx.Supervised import SupervisedModel from compClust.mlx.labelings import Labeling class ANN (Supervised): """Artificial Neural Network (ANN) Supervised Machine Learning Algorithm. """ def __init__(self, dataset=None, labeling=None, parameters=None, model=None): """ANN(dataset, labeling, parameters) or ANN(dataset, model) Creates an Artificial Neural Network (ANN). In the first form, an ANN is created for training. That is, when the ANN is run() it will attempt to learn a mapping from each datum in dataset to the corresponding label in labeling. The network training parameters are configured according to the parameters dictionary. Once training is complete, the learned model (ANNModel) may be retrieved with getModel(). The learned model may be used (when creating new ANN objects) to predict labelings from data. In the second form, an ANN is creating for predicting. That is, when the ANN is run() it will attempt to predict a label for each datum in dataset. The network is completely configured by the given ANNModel. Once prediction is complete (usually quite fast) the predicted labeling may be retrieved with getLabeling(). To determine the operating mode of an ANN object (either 'learn' or 'predict'), call its getMode() method. """ Supervised.__init__(self, dataset, labeling, parameters, model) self.__setDefaultParameters() if self.model is None: self.model = ANNModel(dataset, labeling, parameters['hiddenUnits']) return None def run(self): """run() -> status code Runs the ANN wrapper and returns a status code. If getMode() == 'learn', getModel() will return a trained ANN model. If getMode() == 'predict', getLabeling() will return the predicted target values for the given dataset, based on the given model. """ self.__createTempFiles() self.__writeTempFiles() if self.__run() != 0: status = compClust.mlx.wrapper.WRAPPER_STATUS_ERROR else: status = compClust.mlx.wrapper.WRAPPER_STATUS_DONE self.__readTempFiles() self.__removeTempFiles() return status def validate(self): """validate() -> boolean (0 | 1) Returns 1 if all parameters and environment variables necessary to run the ANN are defined, 0 otherwise. """ error = 0 if compClust.util.Verify.environment_variables_exist( [ 'ANN_COMMAND' ] ): error = 1 return not error def __createTempFiles(self): """__createTempFiles() Creates a temporary directory (e.g. /tmp/uwbp*) and within that files for uwbp .patterns, .weights, .cmd, and .out files. The following are set appropriately: - self.patternsFilename - self.weightsFilename - self.commandFilename - self.outputFilename """ tempfile.tempdir = \ compClust.util.WrapperUtil.create_temporary_directory('ANN') self.patternsFilename = tempfile.mktemp( 'patterns' ) self.weightsFilename = tempfile.mktemp( 'weights' ) self.commandFilename = tempfile.mktemp( 'commands' ) self.outputFilename = tempfile.mktemp( 'output' ) return None def __getArchitectureString(self): """__getArchitectureString() -> string Returns a string describing the architecture of the ANN in self.model. The string, suitable for the uwbp make_standard_net command, is of a list of whitespace separated numbers: inputUnits [number of hidden units in each layer] outputUnits """ arch = [ self.model.numInputUnits ] arch.extend( self.model.hiddenUnits ) arch.append( self.model.numOutputUnits ) return string.join( map(str, arch) ) def __readOutputPredictions(self, stream): """__readOutputPredictions(stream) -> Labeling Reads a uwbp .output file created according to commands written in __writeCommandsToPredict(). The output predictions read are mapped to labelings of self.dataset according to the (reverse) LabelMap of self.model. The labeling is returned. The first and last three lines of the file are ignored. Remaining lines are, in C printf notation (one for each pattern): Acts for pattern%d: (%4.3f ... ) target NONE, predict %d, INCORRECT. The integer (%d) after the string 'predict' correspond to numeric labels stored in the ANNModel's LabelMap. """ reverseLabelMap = self.model.getReverseLabelMap() labeling = Labeling(self.dataset, 'ANN Predictions') row = 0 lines = stream.readlines() lines = lines[3:-3] for line in lines: # # Get prediction number from line and look-up corresponding label in # reverseLabelMap: # # "Acts for pattern%d: (%4.3f ... ) target NONE, predict %d, ..." # 012345678 # fields = string.split(line, ',') prediction = string.strip(fields[1][8:]) if prediction == 'NONE': label = 'NONE' else: prediction = int(prediction) - 1 label = reverseLabelMap[prediction] labeling.addLabelToRow(label, row) row = row + 1 return labeling def __readTempFiles(self): """__readTempFiles() """ if self.mode == 'learn': stream = open(self.weightsFilename, 'r') self.model.weights = self.__readWeights(stream) stream.close() else: stream = open(self.outputFilename, 'r') self.labeling = self.__readOutputPredictions(stream) stream.close() return None def __readWeights(self, stream): """__readWeights(stream) -> Numeric array Reads uwbp network weights from the given stream. The weight values are returned as a one-dimensional Numeric array. """ weights = stream.readlines() weights = map(float, weights) return weights def __removeTempFiles(self): """__removeTempFiles() Removes the temporary directory and files created by __createTempFile(). The self.*Filename variables are cleared. """ for file in os.listdir( tempfile.tempdir ): os.remove( os.path.join(tempfile.tempdir, file) ) os.rmdir(tempfile.tempdir) self.patternsFilename = None self.weightsFilename = None self.commandFilename = None self.outputFilename = None # # Restore tempfile.tempdir to its default. # tempfile.tempdir = self.default_tempdir def __run(self): """__run() -> os.WEXITSTATUS value Runs uwbp by calling operating system services. Returns the uwbp exit status: 0 indicates success, non-zero indicates failure. """ command = "%s %s %s" % ( os.environ['ANN_COMMAND'], self.commandFilename, self.outputFilename ) return os.WEXITSTATUS( os.system(command) ) def __setDefaultParameters(self): """__setDefaultParameters() Creates self.parameters if necessary and assigns reasonable default values for any unset parameters. """ if self.parameters is None: self.parameters = {} parameters = self.parameters parameters.setdefault( 'seed' , 42 ) parameters.setdefault( 'lrate' , .002 ) parameters.setdefault( 'numIterations', 10000 ) parameters.setdefault( 'hiddenUnits' , [] ) return None def __writeCommandsToLearn(self, stream): """__writeCommandsToLearn(stream) Writes a uwbp commands to the given stream from values in the internal parameters dictionary and ANNModel. For a description of uwbp commands, see uwbp/README.vars. The commands written are specific to learning a mapping from a dataset to a labeling. See also __writeCommandsToPredict(). """ seed = self.parameters[ 'seed' ] lrate = self.parameters[ 'lrate' ] nepochs = self.parameters[ 'numIterations' ] architecture = self.__getArchitectureString() stream.write( 'set seed %s\n' % str( seed ) ) stream.write( 'set lrate %s\n' % str( lrate ) ) stream.write( 'set nepochs %s\n' % str( nepochs ) ) stream.write( 'set lflag 1\n' ) stream.write( '\n' ) stream.write( 'set print_confusion_matrix? 0\n' ) stream.write( 'set print_test_results_only? 1\n' ) stream.write( '\n' ) stream.write( 'make_standard_net %s\n' % architecture ) stream.write( '\n' ) stream.write( 'get patterns %s\n' % self.patternsFilename ) stream.write( 'ptrain\n' ) stream.write( '\n' ) stream.write( 'save weights %s\n' % self.weightsFilename ) stream.write( 'quit\n' ) return None def __writeCommandsToPredict(self, stream): """__writeCommandsToPredict(stream) Writes a uwbp commands to the given stream from values in the internal parameters dictionary and ANNModel. For a description of uwbp commands, see uwbp/README.vars. The commands written are specific to predicting a labeling from a dataset. See also __writeCommandsToLearn(). """ seed = self.parameters[ 'seed' ] architecture = self.__getArchitectureString() stream.write( 'set seed %s\n' % str( seed ) ) stream.write( 'set nepochs 1\n' ) stream.write( 'set lflag 0\n' ) stream.write( 'set oflag 1\n' ) stream.write( '\n' ) stream.write( 'set implicit_no_output? 1\n' ) stream.write( 'set print_confusion_matrix? 0\n' ) stream.write( '\n' ) stream.write( 'make_standard_net %s\n' % architecture ) stream.write( '\n' ) stream.write( 'get weights %s\n' % self.weightsFilename ) stream.write( 'get patterns %s\n' % self.patternsFilename ) stream.write( 'strain\n' ) stream.write( '\n' ) stream.write( 'quit\n' ) return None def __writePatternsToLearn(self, stream): """__writePatternsToLearn(stream) Writes uwbp patterns to the given stream. The uwbp pattern format is one training example per line: Pattern name may be any string not containing whitespace. In this case, it is simply 'patternN', where N is the row number in self.dataset. Pattern data are floating-point numbers, one for each feature in the given pattern, i.e. self.dataset row data. Output values are the target (correct) values (integer or real) for each output unit. For now, the number of output units is the same as the number of unique labels. Each unique label is assigned its own output unit. The output for a unit is 1 if its corresponding label is the target value for a particular pattern, 0 otherwise. For example, given three unique labels, 'foo', 'bar', and 'baz' their corresponding output unit values would be: foo -> 1 0 0 bar -> 0 1 0 baz -> 0 0 1 At some point it may be nice to let the end-user control this by specifying their own label -> output unit value mapping. Among other things, this would allow users to let one output unit represent more than one value (e.g. foo = 0 and bar = 1, or real-valued outputs, foo = .3, bar = .6, baz = .9). See also __writePatternsToPredict() """ # # The dictionary labelMap (from self.model) and list outputs are used # to turn labels into sequences of discrete (0 or 1) output unit # values. # # labelMap stores the index into outputs of the 0 that will be # (temporarily) flipped to a 1 for a given label. # labelMap = self.model.labelMap outputs = ['0'] * self.model.numOutputUnits rowLabels = self.labeling.getLabelByRows() ## data = colUnitNormalize( self.dataset.getData() ) data = self.dataset.getData() numRows = self.dataset.getNumRows() for n in range(numRows): pos = labelMap[ rowLabels[n] ] features = map(str, data[n, :]) outputs[pos] = '1' stream.write( 'pattern' ) stream.write( str(n) ) stream.write( '\t' ) stream.write( string.join(features, '\t') ) stream.write( '\t' ) stream.write( string.join(outputs , '\t') ) stream.write( '\n' ) outputs[pos] = '0' return None def __writePatternsToPredict(self, stream): """__writePatternsToPredict(stream) Writes uwbp patterns to the given stream. The uwbp pattern format is one training example per line: Pattern name may be any string not containing whitespace. In this case, it is simply 'patternN', where N is the row number in self.dataset. Pattern data are floating-point numbers, one for each feature in the given pattern, i.e. self.dataset row data. Output values are the target values (integer or real) for each output unit. In 'predict' mode, output values are not required, at least conceptually. However, uwbp always requires them, so zeros (equal to the number of output units) are written. See also __writePatternsToLearn() """ outputs = ['0'] * self.model.numOutputUnits ## data = colUnitNormalize( self.dataset.getData() ) data = self.dataset.getData() numRows = self.dataset.getNumRows() for n in range(numRows): features = map(str, data[n, :]) stream.write( 'pattern' ) stream.write( str(n) ) stream.write( '\t' ) stream.write( string.join(features, '\t') ) stream.write( '\t' ) stream.write( string.join(outputs , '\t') ) stream.write( '\n' ) return None def __writeTempFiles(self): """__writeTempFiles() Writes the uwbp .patterns, .command, and if self.mode == 'predict', .weights files. """ patternsStream = open( self.patternsFilename, 'w' ) commandsStream = open( self.commandFilename , 'w' ) if self.mode == 'learn': self.__writePatternsToLearn( patternsStream ) self.__writeCommandsToLearn( commandsStream ) else: self.__writePatternsToPredict( patternsStream ) self.__writeCommandsToPredict( commandsStream ) weightsStream = open(self.weightsFilename, 'w') self.__writeWeights(weightsStream, self.model.weights) weightsStream.close() patternsStream.close() commandsStream.close() return None def __writeWeights(self, stream, weights): """__writeWeightsFile(stream, weights) Writes uwbp weights to the given stream. The weight values are written one per line. The weights parameter is a list of weight values (reals). """ stream.write( string.join( map(str, weights), '\n' ) ) return None class ANNModel (SupervisedModel): """ANNModel Artificial Neural Network (ANN) Model is currently a convenient container with simple public attributes for the most critical information about a neural network and the model it produces. These attributes are the number of input and output units, a list of integers indicating the number of units in each successive hidden layer (may be empty) and a list to store (synapse) weight values. """ def __init__(self, dataset, labeling, hiddenUnits=None): """ANNModel(dataset, labeling, hiddenUnits=None) -> ANNModel Creates a new ANNModel based on the given dataset, labeling, and list of the number of hidden units in each consecutive layer. For simple perceptrons, hiddenUnits is None. Only the ANN wrapper should created an ANNModel. The dimensionality (number of columns) of the dataset sets the number of input units. The unique labels in labeling is used to create a labelMap and set the number of output units. """ SupervisedModel.__init__(self, labeling) if hiddenUnits is None: hiddenUnits = [] self.numInputUnits = dataset.getNumCols(); self.numOutputUnits = len(self.labelMap); self.hiddenUnits = hiddenUnits; self.weights = []; def colUnitNormalize(data): """colUnitNormalize(data) -> data Returns data where each column is normalized to the range [0, 1]. The normalization is computed for each value by subtracting the minimum column value and dividing by the column range (max - min). Data is, of course, two-dimensional, i.e. a matrix. The same thing could be accomplished with our *View classes. The same amount of work (code) would be required, but there is no apparent gain in readability and a there is a loss in efficiency. """ minimums = Numeric.minimum.reduce(data) maximums = Numeric.maximum.reduce(data) ranges = maximums - minimums return (data - minimums) / ranges # # FIXME: Launcher assumes usage: command , # FIXME: but should now handle targets and models. # # # if __name__ == "__main__": # Launcher.main(sys.argv, ANN()) #