Changeset 6415

Timestamp:

Feb 25, 2009, 11:44:06 AM (16 years ago)

Author:

rwilson

Message:

Added routines to encode/decode arrays of strings (and test).

Location:

branches/numpy/anuga/utilities

Files:

• system_tools.py (modified) (2 diffs)
• test_system_tools.py (modified) (1 diff)

branches/numpy/anuga/utilities/system_tools.py

@@ -6 +6 @@
 import sys
 import os
+import string
 
 def log_to_file(filename, s, verbose=False):
@@ -260 +261 @@
 
 
+################################################################################
+# The following two functions are used to get around a problem with numpy and
+# NetCDF files.  Previously, using Numeric, we could take a list of strings and
+# convert to a Numeric array resulting in this:
+#     Numeric.array(['abc', 'xy']) -> [['a', 'b', 'c'],
+#                                      ['x', 'y', ' ']]
+#
+# However, under numpy we get:
+#     numpy.array(['abc', 'xy']) -> ['abc',
+#                                    'xy']
+#
+# And writing *strings* to a NetCDF file is problematic.
+#
+# The solution is to use these two routines to convert a 1-D list of strings
+# to the 2-D list of chars form and back.  The 2-D form can be written to a 
+# NetCDF file as before.
+#
+# The other option, of inverting a list of tag strings into a dictionary with
+# keys being the unique tag strings and the key value a list of indices of where
+# the tag string was in the original list was rejected because:
+#    1. It's a lot of work
+#    2. We'd have to rewite the I/O code a bit (extra variables instead of one)
+#    3. The code below is fast enough in an I/O scenario
+################################################################################
+
+##
+# @brief Convert 1-D list of strings to 2-D list of chars.
+# @param l 1-dimensional list of strings.
+# @return A 2-D list of 'characters' (1 char strings).
+# @note No checking that we supply a 1-D list.
+def string_to_char(l):
+    '''Convert 1-D list of strings to 2-D list of chars.'''
+
+    maxlen = reduce(max, map(len, l))
+    ll = [x.ljust(maxlen) for x in l]
+    result = []
+    for s in ll:
+        result.append([x for x in s])
+    return result
+
+
+##
+# @brief Convert 2-D list of chars to 1-D list of strings.
+# @param ll 2-dimensional list of 'characters' (1 char strings).
+# @return A 1-dimensional list of strings.
+# @note Each string has had right-end spaces removed.
+def char_to_string(ll):
+    '''Convert 2-D list of chars to 1-D list of strings.'''
+
+    return map(string.rstrip, [''.join(x) for x in ll])
+
+

branches/numpy/anuga/utilities/test_system_tools.py

@@ -183 +183 @@
 
 ################################################################################
+# Test the string_to_char() and char_to_string() utility functions.
+################################################################################
+
+    def test_string_to_char(self):
+        import random
+
+        MAX_CHARS = 10
+        MAX_ENTRIES = 100000
+        A_INT = ord('a')
+        Z_INT = ord('z')
+
+        # generate some random strings in a list, with guaranteed lengths
+        str_list = ['x' * MAX_CHARS]        # make first maximum length
+        for entry in xrange(MAX_ENTRIES):
+            length = random.randint(1, MAX_CHARS)
+            s = ''
+            for c in range(length):
+                s += chr(random.randint(A_INT, Z_INT))
+            str_list.append(s)
+
+        x = string_to_char(str_list)
+        new_str_list = char_to_string(x)
+
+        self.failUnlessEqual(new_str_list, str_list)
+
+
+################################################################################
+# Test the raw I/O to NetCDF files of string data encoded/decoded with 
+# string_to_char() and char_to_string().
+################################################################################
+
+################################################################################
 
 if __name__ == "__main__":
-    suite = unittest.makeSuite(Test_system_tools, 'test')
+    #suite = unittest.makeSuite(Test_system_tools, 'test')
+    suite = unittest.makeSuite(Test_system_tools, 'test_string_to_char')
     runner = unittest.TextTestRunner()
     runner.run(suite)