Transcribe Cantonese Speech to Text: with Code Samples and Automated Batch Processing Techniques

!pip install soundfile
!pip install pydub
!pip install SpeechRecognition
!pip install openai --upgrade

import os
import csv
import wave
import soundfile
import requests
import pandas as pd
from datetime import timedelta
from pydub import AudioSegment
import speech_recognition as sr
from transformers import pipeline
from openai import AzureOpenAI

def stt_Cantonese(audio_file):

    # Write the audio data back to the same file, using the original sample rate and a 16-bit PCM encoding to avoid the ValueError: Audio file could not be read as PCM WAV, AIFF/AIFF-C, or Native FLAC; check if file is corrupted or in another format
    data, samplerate = soundfile.read(audio_file)
    soundfile.write(audio_file, data, samplerate, subtype='PCM_16')

    r = sr.Recognizer()

    with sr.AudioFile(audio_file) as source:
        audio = r.record(source)

    try:
        text = r.recognize_google(audio, language="yue-HK") # yue-HK = Cantonese
        return text
    
    except sr.UnknownValueError:
        print("Google Speech Recognition could not understand audio")
        return None
    except sr.RequestError as e:
        print("Could not request results from Google Speech Recognition service; {0}".format(e))
        return None

# Parameters
client = AzureOpenAI(
  azure_endpoint = "https://hkust.azure-api.net",
  api_version = "2024-06-01",
  api_key = "<your_azure_openai_api_key>" # See here to see how to get your HKUST Azure OpenAI API key: https://digitalhumanities.hkust.edu.hk/tutorials/how-to-use-hkust-azure-openai-api-key-with-python-with-sample-code-and-use-case-examples/
)

def refineCantonese(message):
    try:
        response = client.chat.completions.create(
            model = 'gpt-4o',
            temperature = 1,
            messages = [
                {"role": "system", "content": """
                                            You are an expert in Cantonese. Please perform the following two steps and return the revised text.
                                            1. Correct potential inaccuracies in the Cantonese text in Hong Kong style. For example, in Hong Kong, text such as '噉', should be written as '咁'.
                                            2. Add punctuation to the text.
                                            """},
                {"role": "user", "content": message}
            ]
        )
        return response.choices[0].message.content
    except:
        return None

MODEL_NAME = "alvanlii/whisper-small-cantonese" 
lang = "zh"
device = "cpu"  # or "cuda" if you have a GPU
pipe = pipeline(
    task="automatic-speech-recognition",
    model=MODEL_NAME,
    chunk_length_s=30,
    device=device,
)
pipe.model.config.forced_decoder_ids = pipe.tokenizer.get_decoder_prompt_ids(language=lang, task="transcribe")

HF_API_URL = "https://api-inference.huggingface.co/models/openai/whisper-large-v3"
headers = {"Authorization": "Bearer <your_hugging_face_api_key>"} #You may get your Hugging Face api key here: https://huggingface.co/settings/tokens

def stt_WrittenChi(audio_file):
    with open(audio_file, "rb") as f:
        data = f.read()
    response = requests.post(HF_API_URL, headers=headers, data=data)
    return response.json()["text"]

# Parameters
client = AzureOpenAI(
  azure_endpoint = "https://hkust.azure-api.net",
  api_version = "2024-06-01",
  api_key = "<your_azure_openai_api_key>" # See here to see how to get your HKUST Azure OpenAI API key: https://digitalhumanities.hkust.edu.hk/tutorials/how-to-use-hkust-azure-openai-api-key-with-python-with-sample-code-and-use-case-examples/
)

def Canto_to_Chi_OpenAI(message):
    try:
        response = client.chat.completions.create(
            model = 'gpt-4o',
            temperature = 1,
            messages = [
                {"role": "system", "content": """
                                            Translate the following text from spoken Cantonese text to written language of traditional Chinese text.
                                            """},
                {"role": "user", "content": message}
            ]
        )
        return response.choices[0].message.content
    except:
        return None

# Parameters
client = AzureOpenAI(
  azure_endpoint = "https://hkust.azure-api.net",
  api_version = "2024-06-01",
  api_key = "<your_azure_openai_api_key>" # See here to see how to get your HKUST Azure OpenAI API key: https://digitalhumanities.hkust.edu.hk/tutorials/how-to-use-hkust-azure-openai-api-key-with-python-with-sample-code-and-use-case-examples/
)

def summarize(message):
    try:
        response = client.chat.completions.create(
            model = 'gpt-4o',
            temperature = 1,
            messages = [
                {"role": "system", "content": """
                                            You are a journalist. Summarize the following text in traditional Chinese in a concise manner:
                                            """},
                {"role": "user", "content": message}
            ]
        )
        return response.choices[0].message.content
    except:
        return None

# Set the path to the directory containing the WAV files
path = '.' # current folder together with this ipynb file

# Create an empty list to store the data
data = []

# Loop through all files in the directory
for filename in os.listdir(path):
    if filename.endswith('.wav'):
        # Load the WAV file
        audio = AudioSegment.from_wav(os.path.join(path, filename))
        
        # Get the duration of the audio file
        duration = int(audio.duration_seconds)
        duration = str(timedelta(seconds=duration))

        filename_str = str(filename)

        # Speech (Cantonese) to Text (Spoken Cantonese)
        cantonese_text = stt_Cantonese(filename)

        if cantonese_text is not None:
            # Add punctuation and refine the Spoken Cantonese Text
            cantonese_text = refineCantonese(cantonese_text)
        
        # Add the data to the list
        data.append({
            'filename': filename,
            'duration': duration,
            'cantonese_text': cantonese_text,
        })

# DataFrame
df = pd.DataFrame(data)

# Save to a CSV file
df.to_csv('output_CantoneseText.csv', index=False, encoding='utf-8')
print("Exported in a csv file.")

# Import the refined Cantonese text
df_refined = pd.read_csv('output_CantoneseText.csv', encoding='utf-8')

# Add two new columns
df_refined['writtenChi_text'] = df_refined['cantonese_text_refined'].apply(Canto_to_Chi_OpenAI)
df_refined['summary'] = df_refined['cantonese_text_refined'].apply(summarize)

# Save to a CSV file
df_refined.to_csv('output_CantoneseText_ChiWrittenText.csv', index=False, encoding='utf-8')
print("Exported in a csv file.")