Fix language filter codes (#4841)

* Fix language filter codes

CLD3 returns BCP-47 language identifier, filter settings expect
identifiers in the ISO 639-1 format. Convert between formats,
and exclude duplicate languages from filter choices (zh-CN->zh)

* Fix zh name

app/helpers/settings_helper.rb

@@ -30,6 +30,7 @@ module SettingsHelper
     th: 'ภาษาไทย',
     tr: 'Türkçe',
     uk: 'Українська',
+    zh: '中文',
     'zh-CN': '简体中文',
     'zh-HK': '繁體中文（香港）',
     'zh-TW': '繁體中文（臺灣）',
@@ -39,6 +40,10 @@ module SettingsHelper
     HUMAN_LOCALES[locale]
   end
 
+  def filterable_languages
+    I18n.available_locales.map { |locale| locale.to_s.split('-').first.to_sym }.uniq
+  end
+
   def hash_to_object(hash)
     HashObject.new(hash)
   end

app/lib/language_detector.rb

@@ -20,7 +20,16 @@ class LanguageDetector
   private
 
   def detected_language_code
-    result.language.to_sym if detected_language_reliable?
+    iso6391(result.language).to_sym if detected_language_reliable?
+  end
+
+  def iso6391(bcp47)
+    iso639 = bcp47.split('-').first
+
+    # CLD3 returns grandfathered language code for Hebrew
+    return 'he' if iso639 == 'iw'
+
+    ISO_639.find(iso639).alpha2
   end
 
   def result

app/views/settings/preferences/show.html.haml

@@ -13,7 +13,7 @@
       selected: I18n.locale
 
     = f.input :filtered_languages,
-      collection: I18n.available_locales,
+      collection: filterable_languages,
       wrapper: :with_block_label,
       include_blank: false,
       label_method: lambda { |locale| human_locale(locale) },