Upload parquet to.txt-ALMA-make imatrix.py

parquet to.txt-ALMA-make imatrix.py

@@ -0,0 +1,23 @@
+import pandas as pd
+
+# first need to download from haoranxu/ALMA-R-Preference(https://huggingface.co/datasets/haoranxu/ALMA-R-Preference)
+
+# Parquet to txt
+df = pd.read_parquet('haoranxu-ALMA-R-Preference.parquet')
+print(df.columns)
+# text_column = df[['alma_en', 'alma_zh', 'en', 'gpt4_en', 'gpt4_zh', 'zh']]
+# text_column = df[['en', 'zh']]
+
+
+
+# save txt
+with open('haoranxu-ALMA-R-Preference-en-zh--zh-en.txt', 'w', encoding='utf-8') as f:
+    for item in df['translation']:
+        en_text = item.get('en')
+        zh_text = item.get('zh')
+        if en_text and zh_text:  # check 'en' and 'zh'
+            f.write(f"English: {en_text}\nChinese: {zh_text}\n\n")
+            f.write(f"Chinese: {zh_text}\nEnglish: {en_text}\n\n")
+            
+            
+# then u can use it to make your language imatrix.dat