# Python script for initial FASTA chunking
import os
from Bio import SeqIO

def chunk_fasta(input_fasta_path, output_dir, chunk_size_gb=1):
    """Splits a large FASTA file into smaller, manageable chunks based on approximate file size."""
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    record_buffer = []
    current_chunk_size = 0
    chunk_count = 0
    max_bytes_per_chunk = chunk_size_gb * (1024**3) # Convert GB to bytes

    print(f"Initiating chunking of {input_fasta_path} into {chunk_size_gb}GB segments...")

    with open(input_fasta_path, "r") as infile:
        for record in SeqIO.parse(infile, "fasta"):
            record_buffer.append(record)
            # Approximate size of record (ID + sequence + newline chars)
            current_chunk_size += len(record.id.encode('utf-8')) + len(str(record.seq).encode('utf-8')) + 2 # Approx newlines

            if current_chunk_size >= max_bytes_per_chunk:
                chunk_filename = os.path.join(output_dir, f"chunk_{chunk_count:04d}.fasta")
                with open(chunk_filename, "w") as outfile:
                    SeqIO.write(record_buffer, outfile, "fasta")
                print(f"Chunk {chunk_count:04d} saved: {chunk_filename} ({current_chunk_size / (1024**3):.2f} GB)")
                record_buffer = []
                current_chunk_size = 0
                chunk_count += 1
    
    # Write any remaining records in the buffer
    if record_buffer:
        chunk_filename = os.path.join(output_dir, f"chunk_{chunk_count:04d}.fasta")
        with open(chunk_filename, "w") as outfile:
            SeqIO.write(record_buffer, outfile, "fasta")
        print(f"Final chunk {chunk_count:04d} saved: {chunk_filename} ({current_chunk_size / (1024**3):.2f} GB)")

# Example usage:
# input_file = "large_protein_sequences.fasta"
# output_directory = "fasta_chunks"
# chunk_fasta(input_file, output_directory, chunk_size_gb=2)

# Python script for cleaning FASTA sequences within a chunk
from Bio import SeqIO

def clean_sequence(seq_record):
    """Applies a series of cleaning operations to a single SeqRecord."""
    sequence = str(seq_record.seq).upper()
    # Define allowed characters for protein sequences (standard 20 amino acids + B, J, O, U, X, Z)
    # Adjust this set if handling DNA/RNA or non-standard amino acids
    allowed_chars = set("ACDEFGHIKLMNPQRSTVWXYBJOXZ-") # '-' for gaps, often present
    
    # 1. Remove non-standard characters
    cleaned_sequence = ''.join(char for char in sequence if char in allowed_chars)

    # 2. Handle ambiguous bases/amino acids if desired (e.g., replace 'X' with special token or remove)
    # For this guide, we'll keep 'X' as it's common for unknown/ambiguous residues in protein data.
    # If strict, consider: cleaned_sequence = cleaned_sequence.replace('X', '') or replace with a placeholder.

    # 3. Filter by length (e.g., minimum sequence length)
    min_len = 10  # Define minimum length for valid protein sequences
    if len(cleaned_sequence) < min_len:
        return None # Indicate sequence should be discarded

    # Update the sequence record with the cleaned sequence
    seq_record.seq = type(seq_record.seq)(cleaned_sequence)
    return seq_record

def process_chunk_for_cleaning(input_fasta_path, output_fasta_path):
    """Reads a FASTA chunk, cleans sequences, and writes valid ones to a new file."""
    valid_records = []
    total_records = 0
    discarded_records = 0
    print(f"Cleaning chunk: {input_fasta_path}")

    with open(input_fasta_path, "r") as infile:
        for record in SeqIO.parse(infile, "fasta"):
            total_records += 1
            cleaned_record = clean_sequence(record)
            if cleaned_record is not None:
                valid_records.append(cleaned_record)
            else:
                discarded_records += 1
    
    with open(output_fasta_path, "w") as outfile:
        SeqIO.write(valid_records, outfile, "fasta")
    
    print(f"Cleaning complete for {input_fasta_path}. Total: {total_records}, Valid: {len(valid_records)}, Discarded: {discarded_records}")
    return len(valid_records), discarded_records

# Example usage for a single chunk:
# input_chunk = "fasta_chunks/chunk_0000.fasta"
# output_cleaned_chunk = "cleaned_fasta_chunks/chunk_0000_cleaned.fasta"
# process_chunk_for_cleaning(input_chunk, output_cleaned_chunk)

# Python script to convert cleaned FASTA chunks to Hugging Face-compatible JSONL
import os
import json
from Bio import SeqIO
from datasets import Dataset, DatasetDict

def fasta_to_jsonl(input_fasta_path, output_jsonl_path, sequence_key="sequence", id_key="id"):
    """Converts a FASTA file into a JSONL file, suitable for Hugging Face datasets."""
    print(f"Converting {input_fasta_path} to JSONL: {output_jsonl_path}")
    total_converted = 0
    with open(output_jsonl_path, "w") as outfile:
        with open(input_fasta_path, "r") as infile:
            for record in SeqIO.parse(infile, "fasta"):
                data = {
                    id_key: str(record.id),
                    sequence_key: str(record.seq)
                }
                outfile.write(json.dumps(data) + '\n')
                total_converted += 1
    print(f"Successfully converted {total_converted} records to JSONL.")
    return output_jsonl_path

def create_huggingface_dataset(jsonl_files_list, cache_dir="./hf_dataset_cache"):
    """Loads multiple JSONL files into a streaming Hugging Face Dataset."""
    # Ensure the cache directory exists
    if not os.path.exists(cache_dir):
        os.makedirs(cache_dir)
        
    print(f"Loading {len(jsonl_files_list)} JSONL files into Hugging Face Dataset...")
    # Hugging Face 'datasets' library can directly load from multiple JSONL files
    # Use streaming mode for large datasets to avoid loading everything into RAM
    # 'line_by_line=True' is crucial for JSONL format
    dataset = Dataset.from_file(jsonl_files_list, split='train', streaming=True, cache_dir=cache_dir)
    # Or, if you want to load from a directory containing many JSONL files:
    # dataset = load_dataset('json', data_files=jsonl_files_list, split='train', streaming=True)

    # Consider a train-validation split if not already done
    # For this example, we assume all files are 'train' for simplicity.
    # dataset_dict = dataset.train_test_split(test_size=0.01) 
    print("Hugging Face Dataset created successfully in streaming mode.")
    return dataset

# Example workflow for a directory of cleaned FASTA chunks:
# cleaned_fasta_dir = "cleaned_fasta_chunks"
# jsonl_output_dir = "hf_jsonl_datasets"
# os.makedirs(jsonl_output_dir, exist_ok=True)

# all_jsonl_files = []
# for fasta_file in os.listdir(cleaned_fasta_dir):
#     if fasta_file.endswith(".fasta"):
#         input_path = os.path.join(cleaned_fasta_dir, fasta_file)
#         output_path = os.path.join(jsonl_output_dir, fasta_file.replace(".fasta", ".jsonl"))
#         fasta_to_jsonl(input_path, output_path)
#         all_jsonl_files.append(output_path)

# hf_streaming_dataset = create_huggingface_dataset(all_jsonl_files)
# print(hf_streaming_dataset)
# # You can now iterate over hf_streaming_dataset or apply tokenization/mapping functions.
# # Example: print(next(iter(hf_streaming_dataset)))

# Python setup for parallel processing of chunks using joblib
from joblib import Parallel, delayed
import os

# Assume process_chunk_for_cleaning and fasta_to_jsonl functions are defined as above
# And a directory structure: fasta_chunks/ -> cleaned_fasta_chunks/ -> hf_jsonl_datasets/

def run_full_workflow_for_chunk(chunk_path, cleaned_output_dir, jsonl_output_dir):
    """Combines cleaning and JSONL conversion for a single chunk."""
    chunk_filename = os.path.basename(chunk_path)
    cleaned_fasta_path = os.path.join(cleaned_output_dir, chunk_filename.replace(".fasta", "_cleaned.fasta"))
    jsonl_path = os.path.join(jsonl_output_dir, chunk_filename.replace(".fasta", ".jsonl"))
    
    # 1. Clean the chunk
    valid_count, discarded_count = process_chunk_for_cleaning(chunk_path, cleaned_fasta_path)
    
    # 2. Convert cleaned chunk to JSONL, only if valid sequences exist
    if valid_count > 0:
        fasta_to_jsonl(cleaned_fasta_path, jsonl_path)
        return jsonl_path # Return path of the newly created JSONL file
    else:
        print(f"No valid sequences in {chunk_path}, skipping JSONL conversion.")
        return None

def orchestrate_parallel_processing(fasta_chunks_dir, cleaned_output_dir, jsonl_output_dir, n_jobs=-1):
    """Orchestrates parallel processing of all FASTA chunks."""
    os.makedirs(cleaned_output_dir, exist_ok=True)
    os.makedirs(jsonl_output_dir, exist_ok=True)

    fasta_files = [os.path.join(fasta_chunks_dir, f) for f in os.listdir(fasta_chunks_dir) if f.endswith(".fasta")]

    print(f"Starting parallel processing across {len(fasta_files)} chunks with {n_jobs} jobs.")
    results = Parallel(n_jobs=n_jobs, verbose=10)(
        delayed(run_full_workflow_for_chunk)(chunk_file, cleaned_output_dir, jsonl_output_dir)
        for chunk_file in fasta_files
    )
    
    # Filter out None results (from chunks with no valid sequences)
    all_jsonl_files = [res for res in results if res is not None]
    print("Parallel processing complete.")
    return all_jsonl_files

# Example usage:
# fasta_chunks_source = "fasta_chunks"
# cleaned_chunks_dest = "cleaned_fasta_chunks"
# jsonl_datasets_dest = "hf_jsonl_datasets"
# final_jsonl_paths = orchestrate_parallel_processing(fasta_chunks_source, cleaned_chunks_dest, jsonl_datasets_dest, n_jobs=4)
# print(f"Total JSONL files prepared: {len(final_jsonl_paths)}")
# hf_streaming_dataset = create_huggingface_dataset(final_jsonl_paths) # Assuming create_huggingface_dataset is imported/defined