Language lookup improvements

git-svn-id: svn+ssh://svn.gna.org/svn/gnustep/libs/base/trunk@35275 72102866-910b-0410-8b05-ffd578937521

ChangeLog

@@ -3,6 +3,8 @@
         * Source/NSBundle.m: Fixes for bug #34815 ... map between old and
         new style language names and use the best language specific
         resource available.
+        * Resources/Languages/Locale.canonical: Add mappings from three
+        letter language names to two letter names where they exist.
 
 2012-07-08  Richard Frith-Macdonald <rfm@gnu.org>
 

Resources/Languages/Locale.canonical

@@ -388,4 +388,189 @@
   Zhuang = za;
   ZimbabweEnglish = en_ZW;
   Zulu = zu;
+  aar = aa;
+  abk = ab;
+  afr = af;
+  alb = sq;
+  amh = am;
+  ara = ar;
+  arm = hy;
+  asm = as;
+  ave = ae;
+  aym = ay;
+  aze = az;
+  bak = ba;
+  baq = eu;
+  bel = be;
+  ben = bn;
+  bih = bh;
+  bis = bi;
+  bod = bo;
+  bos = bs;
+  bre = br;
+  bul = bg;
+  bur = my;
+  cat = ca;
+  ces = cs;
+  cha = ch;
+  che = ce;
+  chi = zh;
+  chu = cu;
+  chv = cv;
+  cor = kw;
+  cos = co;
+  cym = cy;
+  cze = cs;
+  dan = da;
+  deu = de;
+  dut = nl;
+  dzo = dz;
+  ell = el;
+  eng = en;
+  epo = eo;
+  est = et;
+  eus = eu;
+  fao = fo;
+  fas = fa;
+  fij = fj;
+  fin = fi;
+  fra = fr;
+  fre = fr;
+  fry = fy;
+  geo = ka;
+  ger = de;
+  gla = gd;
+  gle = ga;
+  glg = gl;
+  glv = gv;
+  gre = el;
+  grn = gn;
+  guj = gu;
+  hau = ha;
+  heb = he;
+  her = hz;
+  hin = hi;
+  hmo = ho;
+  hrv = hr;
+  hun = hu;
+  hye = hy;
+  ice = is;
+  iku = iu;
+  ile = ie;
+  ina = ia;
+  ind = id;
+  ipk = ik;
+  isl = is;
+  ita = it;
+  jav = jw;
+  jaw = jw;
+  jpn = ja;
+  kal = kl;
+  kan = kn;
+  kas = ks;
+  kat = ka;
+  kaz = kk;
+  khm = km;
+  kik = ki;
+  kin = rw;
+  kir = ky;
+  kom = kv;
+  kor = ko;
+  kua = kj;
+  kur = ku;
+  lao = lo;
+  lat = la;
+  lav = lv;
+  lin = ln;
+  lit = lt;
+  ltz = lb;
+  mac = mk;
+  mah = mh;
+  mal = ml;
+  mao = mi;
+  mar = mr;
+  may = ms;
+  mkd = mk;
+  mlg = mg;
+  mlt = mt;
+  mol = mo;
+  mon = mn;
+  mri = mi;
+  msa = ms;
+  mya = my;
+  nau = na;
+  nav = nv;
+  nbl = nr;
+  nde = nd;
+  ndo = ng;
+  nep = ne;
+  nld = nl;
+  nno = nn;
+  nob = nb;
+  nor = no;
+  nya = ny;
+  oci = oc;
+  ori = or;
+  orm = om;
+  oss = os;
+  pan = pa;
+  per = fa;
+  pli = pi;
+  pol = pl;
+  por = pt;
+  pus = ps;
+  que = qu;
+  roh = rm;
+  ron = ro;
+  rum = ro;
+  run = rn;
+  rus = ru;
+  sag = sg;
+  san = sa;
+  scc = sr;
+  scr = hr;
+  sin = si;
+  slk = sk;
+  slo = sk;
+  slv = sl;
+  sme = se;
+  smo = sm;
+  sna = sn;
+  snd = sd;
+  som = so;
+  sot = st;
+  spa = es;
+  sqi = sq;
+  srd = sc;
+  srp = sr;
+  ssw = ss;
+  sun = su;
+  swa = sw;
+  swe = sv;
+  tah = ty;
+  tam = ta;
+  tat = tt;
+  tel = te;
+  tgk = tg;
+  tgl = tl;
+  tha = th;
+  tib = bo;
+  tsn = tn;
+  tso = ts;
+  tuk = tk;
+  tur = tr;
+  twi = tw;
+  uig = ug;
+  ukr = uk;
+  urd = ur;
+  uzb = uz;
+  vie = vi;
+  vol = vo;
+  wel = cy;
+  wol = wo;
+  xho = xh;
+  yid = yi;
+  zha = za;
+  zho = zh;
+  zul = zu;
 }

Resources/Languages/README

@@ -1,3 +1,8 @@
+The file Locale.aliases is used to map common language specifications to
+old-style long names.
+
+The file Locale.canonical is used to map old style names and three letter
+ISO-639-2 names to the preferred tewo letter ISO-639-1 names.
 
 Language files which contain non-ascii characters should either be properly
 marked unicode files (UTF-8 with a leading Byte Order Mark or UTF-16 with a
@@ -13,14 +18,14 @@ in the default encoding used by your system.
 For example, to edit the French language file so that it contains an ascii
 propertly list with \u escape sequeneces, you could do -
 
-cvtenc -EscapeIn yes French > tmpfile
+cvtenc -EscapeIn yes fr > tmpfile
 vi tmpfile
-cvtenc -EscapeOut yes tmpfile > French
+cvtenc -EscapeOut yes tmpfile > fr
 rm tmpfile
 
 A common case is where you have a file in UTF-8, but the Byte Order Mark is
 missing, so you need to fix it up ...
 
 cvtenc -Encoding 'UNICODE UTF-8' FileWithMissingBOM >tmpFile
-cvtenc -EscapeOut yes tmpFile > French
+cvtenc -EscapeOut yes tmpFile > fr
 

Source/NSBundle.m

@@ -75,10 +75,11 @@ manager()
   return mgr;
 }
 
-static NSDictionary     *alternativeLanguageMap = nil;
+static NSDictionary     *langAliases = nil;
+static NSDictionary     *langCanonical = nil;
 
 /* Map a language name to any alternative versions.   This function should
- * return an array of alternative language/localisation directry names in
+ * return an array of alternative language/localisation directory names in
  * the preferred order of precedence (ie resources in the directories named
  * earlier in the array are to be preferred to those in directories named
  * later).
@@ -88,91 +89,85 @@ static NSDictionary     *alternativeLanguageMap = nil;
  * in the United States region).
  */
 static NSArray *
-altLang(NSString *lang)
+altLang(NSString *full)
 {
-  if (lang)
+  NSMutableArray        *a = nil;
+
+  if (nil != full)
     {
-      NSArray   *a;
+      NSString  *alias = nil;
+      NSString  *canon = nil;
+      NSString  *lang = nil;
+      NSString  *dialect = nil;
+      NSString  *region = nil;
       NSRange   r;
 
-      r = [lang rangeOfString: @"-"];
+      alias = [langAliases objectForKey: full];
-      if (r.length > 0)
+      if (nil == alias)
         {
-          NSString      *full = lang;
+          canon = [langCanonical objectForKey: full];
-
+          if (nil != canon)
-          lang = [full substringToIndex: r.location];
-          r = [lang rangeOfString: @"_"];
-          if (r.length > 0)
             {
-              NSString  *national = [full substringToIndex: r.location];
+              alias = [langAliases objectForKey: canon];
-
-              // language-dialect_region
-              a = [alternativeLanguageMap objectForKey: lang];
-              if (nil == a)
-                {
-                  return [NSArray arrayWithObjects: full, national, lang, nil];
-                }
-              else
-                {
-                  NSMutableArray    *m = [a mutableCopy];
-
-                  [m insertObject: full atIndex: 0];
-                  [m insertObject: national atIndex: 0];
-                  return [m autorelease];
-                }
             }
-          else
+          if (nil == alias)
             {
-              // language-dialect
+              alias = full;
-              a = [alternativeLanguageMap objectForKey: lang];
-              if (nil == a)
-                {
-                  return [NSArray arrayWithObjects: full, lang, nil];
-                }
-              else
-                {
-                  NSMutableArray    *m = [a mutableCopy];
-
-                  [m insertObject: full atIndex: 0];
-                  return [m autorelease];
-                }
             }
         }
+      canon = [langCanonical objectForKey: alias];
+      if (nil == canon)
+        {
+          canon = [langCanonical objectForKey: full];
+          if (nil == canon)
+            {
+              canon = full;
+            }
+        }
+
+      if ([canon rangeOfString: @"-"].length > 1)
+        {
+          dialect = [canon substringFromIndex: NSMaxRange(r)];
+          lang = [canon substringToIndex: r.location];
+          if ([dialect rangeOfString: @"_"].length > 1)
+            {
+              region = [dialect substringFromIndex: NSMaxRange(r)];
+              dialect = [dialect substringToIndex: r.location];
+            }
+        }
+      else if ([canon rangeOfString: @"_"].length > 1)
+        {
+          region = [canon substringFromIndex: NSMaxRange(r)];
+          lang = [canon substringToIndex: r.location];
+        }
       else
         {
-          NSString  *full = lang;
+          lang = canon;
+        }
 
-          r = [lang rangeOfString: @"_"];
+      a = [NSMutableArray arrayWithCapacity: 5];
-          if (r.length > 0)
+      if (nil != dialect && nil != region)
-            {
+        {
-              // language_region
+          [a addObject: [NSString stringWithFormat: @"%@-%@_%@",
-              lang = [full substringToIndex: r.location];
+            lang, dialect, region]];
-              a = [alternativeLanguageMap objectForKey: lang];
+        }
-              if (nil == a)
+      if (nil != dialect)
-                {
+        {
-                  return [NSArray arrayWithObjects: full, lang, nil];
+          [a addObject: [NSString stringWithFormat: @"%@-%@",
-                }
+            lang, dialect]];
-              else
+        }
-                {
+      if (nil != region)
-                  NSMutableArray    *m = [a mutableCopy];
+        {
-
+          [a addObject: [NSString stringWithFormat: @"%@_%@",
-                  [m insertObject: full atIndex: 0];
+            lang, region]];
-                  return [m autorelease];
+        }
-                }
+      [a addObject: lang];
-            }
+      if (NO == [a containsObject: alias])
-          else
+        {
-            {
+          [a addObject: alias];
-              // language
-              a = [alternativeLanguageMap objectForKey: lang];
-              if (nil == a)
-                {
-                  return [NSArray arrayWithObject: lang];
-                }
-              return a;
-            }
         }
     }
-  return nil;
+  return a;
 }
 
 static NSLock *pathCacheLock = nil;
@@ -1066,6 +1061,7 @@ _bundle_load_callback(Class theClass, struct objc_category *theCategory)
   if (self == [NSBundle class])
     {
       extern const char	*GSPathHandling(const char *);
+      NSString          *file;
       const char	*mode;
       NSDictionary	*env;
       NSString		*str;
@@ -1075,25 +1071,41 @@ _bundle_load_callback(Class theClass, struct objc_category *theCategory)
       mode = GSPathHandling("right");
       _emptyTable = RETAIN([NSDictionary dictionary]);
 
-      /* Check ... I think this is all the languages we traditionally
+      /* Create basic mapping dictionaries for bootstrapping and
-       * support ... but maybe we need others.
+       * for use if the full ductionaries can't be loaded from the
+       * gnustep-base library resource bundle.
        */
-      alternativeLanguageMap = [[NSDictionary alloc] initWithObjectsAndKeys:
+      langAliases = [[NSDictionary alloc] initWithObjectsAndKeys:
-        [NSArray arrayWithObjects: @"zh", @"Chinese", nil], @"Chinese",
+        @"German", @"de",
-        [NSArray arrayWithObjects: @"en", @"English", nil], @"English",
+        @"English", @"en",
-        [NSArray arrayWithObjects: @"eo", @"Esperanto", nil], @"Esperanto",
+        @"Esperanto", @"eo",
-        [NSArray arrayWithObjects: @"fr", @"French", nil], @"French",
+        @"Spanish", @"es",
-        [NSArray arrayWithObjects: @"de", @"German", nil], @"German",
+        @"French", @"fr",
-        [NSArray arrayWithObjects: @"it", @"Italian", nil], @"Italian",
+        @"Italian", @"it",
-        [NSArray arrayWithObjects: @"ko", @"Korean", nil], @"Korean",
+        @"Korean", @"ko",
-        [NSArray arrayWithObjects: @"zh", @"Chinese", nil], @"zh",
+        @"TraditionalChinese", @"zh",
-        [NSArray arrayWithObjects: @"en", @"English", nil], @"en",
+        nil];
-        [NSArray arrayWithObjects: @"eo", @"Esperanto", nil], @"eo",
+      langCanonical = [[NSDictionary alloc] initWithObjectsAndKeys:
-        [NSArray arrayWithObjects: @"fr", @"French", nil], @"fr",
+        @"en", @"English",
-        [NSArray arrayWithObjects: @"de", @"German", nil], @"de",
+        @"en", @"eng",
-        [NSArray arrayWithObjects: @"it", @"Italian", nil], @"it",
+        @"ep", @"Esperanto",
-        [NSArray arrayWithObjects: @"ko", @"Korean", nil], @"ko",
+        @"ep", @"epo",
-        [NSArray arrayWithObjects: @"es", @"Spanish", nil], @"es",
+        @"ep", @"epo",
+        @"fr", @"French",
+        @"fr", @"fra",
+        @"fr", @"fre",
+        @"de", @"German",
+        @"de", @"ger",
+        @"de", @"deu",
+        @"it", @"Italian",
+        @"it", @"ita",
+        @"ko", @"Korean",
+        @"ko", @"kir",
+        @"sp", @"Spanish",
+        @"sp", @"spa",
+        @"zh", @"TraditionalChinese",
+        @"zh", @"chi",
+        @"zh", @"zho",
         nil];
 
       /* Initialise manager here so it's thread-safe.
@@ -1138,6 +1150,43 @@ _bundle_load_callback(Class theClass, struct objc_category *theCategory)
 
       _gnustep_bundle = RETAIN([self bundleForLibrary: @"gnustep-base"
 					      version: _base_version]);
+
+      /* The Locale aliases map converts canonical names to old-style names
+       */
+      file = [_gnustep_bundle pathForResource: @"Locale"
+                                       ofType: @"aliases"
+                                  inDirectory: @"Languages"];
+      if (file != nil)
+        {
+          NSDictionary  *d;
+
+          d = [[NSDictionary alloc] initWithContentsOfFile: file];
+          if ([d count] > 0)
+            {
+              ASSIGN(langAliases, d);
+            }
+          [d release];
+        }
+
+      /* The Locale canonical map converts old-style names to ISO 639 names
+       * and converts ISO 639-2 names to the preferred ISO 639-1 names where
+       * an ISO 639-1 name exists.
+       */
+      file = [_gnustep_bundle pathForResource: @"Locale"
+                                       ofType: @"canonical"
+                                  inDirectory: @"Languages"];
+      if (file != nil)
+        {
+          NSDictionary  *d;
+
+          d = [[NSDictionary alloc] initWithContentsOfFile: file];
+          if ([d count] > 0)
+            {
+              ASSIGN(langCanonical, d);
+            }
+          [d release];
+        }
+
 #if 0
       _loadingBundle = [self mainBundle];
       handle = objc_open_main_module(stderr);

Tools/GNUmakefile

@@ -116,6 +116,7 @@ AUTOGSDOC=./$(GNUSTEP_OBJ_DIR_NAME)/autogsdoc
 include Makefile.preamble
 
 include $(GNUSTEP_MAKEFILES)/tool.make
+#include $(GNUSTEP_MAKEFILES)/test-tool.make
 include $(GNUSTEP_MAKEFILES)/ctool.make
 include $(GNUSTEP_MAKEFILES)/aggregate.make
 

Tools/locale_alias.m

@@ -59,9 +59,8 @@ loc_read_file(const char *dir, const char *file)
 
   language[0] = '\0';
   country[0] = '\0';
-  while (1)
+  while (NULL != fgets(buf, MAXSTRING, fp))
     {
-      fgets(buf, MAXSTRING, fp);
       if (strstr(buf, "anguage") != NULL)
 	{
 	  sscanf(&buf[2], "%s", language);
@@ -131,7 +130,7 @@ main(int argc, char *argv[])
 
   /* Write out a skeleton file from the current locale */
   dict = GSDomainFromDefaultLocale();
-  lang = GSLanguageFromLocale(GSSetLocale(NULL));
+  lang = GSLanguageFromLocale(GSSetLocale(0,NULL));
   if (lang == nil)
     lang = @"Locale";
   if (dict)