#### R Script Example: statistical_analysis.R

# Ensure the script is robust for automation:
# 1. Use absolute paths for input/output files.
# 2. Implement argument parsing for flexibility.
# 3. Include basic error handling and logging.

# Install necessary packages if not already installed (uncomment if needed)
# if (!requireNamespace("optparse", quietly = TRUE)) install.packages("optparse")
# if (!requireNamespace("data.table", quietly = TRUE)) install.packages("data.table")

library(optparse)
library(data.table)

# Define command line options
option_list = list(
  make_option(c("-i", "--input"), type="character", default=NULL, 
              help="Path to input CSV file (e.g., /data/raw_data.csv)", metavar="file"),
  make_option(c("-o", "--output"), type="character", default=NULL, 
              help="Path for output CSV file (e.g., /results/processed_data.csv)", metavar="file"),
  make_option(c("-l", "--logfile"), type="character", default="/tmp/r_script.log", 
              help="Path for log file", metavar="file"),
  make_option(c("-a", "--analysis_type"), type="character", default="default", 
              help="Type of analysis to perform (e.g., 't_test', 'linear_model')", metavar="string")
);

opt_parser = OptionParser(option_list=option_list);
opt = parse_args(opt_parser);

# Setup logging function
log_message <- function(msg, level = "INFO") {
  timestamp <- format(Sys.time(), "%Y-%m-%d %H:%M:%S")
  message_to_log <- paste0("[", timestamp, "] ", "[", level, "] ", msg, "\n")
  cat(message_to_log)
  cat(message_to_log, file = opt$logfile, append = TRUE)
}

# --- Main Analysis Logic ---

log_message("Starting R statistical analysis.")

# Validate inputs
if (is.null(opt$input) || !file.exists(opt$input)) {
  log_message(paste0("Error: Input file '", opt$input, "' not found or not specified."), "ERROR")
  stop("Missing or invalid input file.")
}
if (is.null(opt$output)) {
  log_message("Error: Output file path not specified.", "ERROR")
  stop("Missing output file path.")
}

tryCatch({
  log_message(paste0("Reading input data from: ", opt$input))
  data <- fread(opt$input) # Using data.table for efficient reading
  log_message(paste0("Data loaded successfully. Dimensions: ", nrow(data), " rows, ", ncol(data), " columns."))
  
  # Perform analysis based on type
  if (opt$analysis_type == "t_test") {
    log_message("Performing t-test analysis.")
    # Example: Perform a t-test on two columns (assuming 'Group' and 'Value' exist)
    # Check if required columns exist
    if (!all(c("Group", "Value") %in% names(data))) {
      log_message("Error: Columns 'Group' and 'Value' required for t-test not found.", "ERROR")
      stop("Missing columns for t-test.")
    }
    
    group1 <- data[Group == 'A', Value]
    group2 <- data[Group == 'B', Value]
    result <- t.test(group1, group2)
    processed_data <- data.table( 
      Statistic = result$statistic, 
      P_Value = result$p.value, 
      Mean_Group_A = mean(group1), 
      Mean_Group_B = mean(group2)
    )
    log_message("T-test completed.")
  } else if (opt$analysis_type == "linear_model") {
    log_message("Performing linear model analysis.")
    # Example: Perform a linear model (assuming 'Response' and 'Predictor' exist)
    if (!all(c("Response", "Predictor") %in% names(data))) {
      log_message("Error: Columns 'Response' and 'Predictor' required for linear model not found.", "ERROR")
      stop("Missing columns for linear model.")
    }
    result <- lm(Response ~ Predictor, data = data)
    processed_data <- data.table(summary(result)$coefficients)
    log_message("Linear model completed.")
  } else {
    log_message(paste0("Warning: Unknown analysis type '", opt$analysis_type, "'. Returning raw data."), "WARN")
    processed_data <- data # Default: return raw data if analysis type is unknown
  }
  
  log_message(paste0("Writing results to: ", opt$output))
  fwrite(processed_data, opt$output) # Using data.table for efficient writing
  log_message("R script finished successfully.")
  
}, error = function(e) {
  log_message(paste0("An error occurred during R script execution: ", e$message), "ERROR")
  stop(e$message) # Re-throw the error after logging it
})

#### Python Script Example: orchestrate_r_job.py

# This Python script orchestrates the R script execution.
# It handles:
# 1. Command-line argument parsing for dynamic inputs.
# 2. Calling the R script using subprocess.
# 3. Checking R script's exit status.
# 4. Logging its own actions.
# 5. Basic error handling.

import subprocess
import argparse
import os
import sys
import datetime

# --- Configuration Variables ---
R_SCRIPT_PATH = "/path/to/your/statistical_analysis.R"  # <--- UPDATE THIS ABSOLUTE PATH
PYTHON_LOG_DIR = "/tmp/python_logs" # Directory for Python's own logs

# Ensure log directory exists
os.makedirs(PYTHON_LOG_DIR, exist_ok=True)

def log_message(msg, level="INFO"):
    timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    log_file_path = os.path.join(PYTHON_LOG_DIR, "orchestrator.log")
    message_to_log = f"[{timestamp}] [{level}] {msg}"
    print(message_to_log) # Print to stdout/stderr
    with open(log_file_path, "a") as f:
        f.write(message_to_log + "\n")

def main():
    parser = argparse.ArgumentParser(
        description="Orchestrate R statistical analysis script."
    )
    parser.add_argument(
        "--input_file", 
        type=str, 
        required=True, 
        help="Path to the input CSV data file."
    )
    parser.add_argument(
        "--output_file", 
        type=str, 
        required=True, 
        help="Path for the output CSV results file."
    )
    parser.add_argument(
        "--r_log_file", 
        type=str, 
        default="/tmp/r_script_run.log", # Default R log file path for this run
        help="Path for the R script's log file."
    )
    parser.add_argument(
        "--analysis_type", 
        type=str, 
        default="t_test", 
        help="Type of analysis for the R script (e.g., 't_test', 'linear_model')."
    )

    args = parser.parse_args()
    
    log_message(f"Starting Python orchestration for R script: {R_SCRIPT_PATH}")
    log_message(f"Input: {args.input_file}, Output: {args.output_file}, R Log: {args.r_log_file}, Analysis Type: {args.analysis_type}")

    # Construct the R command
    # Ensure Rscript is in the system's PATH or provide its absolute path
    r_command = [
        "Rscript",
        R_SCRIPT_PATH,
        "--input", args.input_file,
        "--output", args.output_file,
        "--logfile", args.r_log_file,
        "--analysis_type", args.analysis_type
    ]

    try:
        # Execute the R script as a subprocess
        # capture_output=True captures stdout/stderr into result.stdout/stderr
        # text=True decodes stdout/stderr as text
        log_message(f"Executing R command: {' '.join(r_command)}")
        result = subprocess.run(
            r_command, 
            capture_output=True, 
            text=True, 
            check=True # Raise CalledProcessError if the command returns a non-zero exit code
        )
        log_message("R script execution successful.")
        if result.stdout:
            log_message(f"R script STDOUT:\n{result.stdout}")
        if result.stderr:
            log_message(f"R script STDERR:\n{result.stderr}", "WARNING")

        log_message(f"R script output written to: {args.output_file}")
        log_message("Python orchestration completed successfully.")

    except subprocess.CalledProcessError as e:
        log_message(f"R script execution failed with exit code {e.returncode}.", "ERROR")
        log_message(f"R script STDOUT:\n{e.stdout}", "ERROR")
        log_message(f"R script STDERR:\n{e.stderr}", "ERROR")
        log_message("Python orchestration failed.", "ERROR")
        sys.exit(1) # Signal failure to cron
    except FileNotFoundError:
        log_message(f"Error: 'Rscript' command not found or R_SCRIPT_PATH invalid. Ensure R is installed and in PATH.", "ERROR")
        sys.exit(1)
    except Exception as e:
        log_message(f"An unexpected error occurred during Python orchestration: {e}", "ERROR")
        sys.exit(1)

if __name__ == "__main__":
    main()

#### Example: Crontab Entry for Scheduling

# 1. Open your crontab for editing:
#    crontab -e

# 2. Add the following line (replace paths and ensure environment is set):

# Example Crontab Entry:
# M H D M W  command_to_be_executed
# * * * * *  /usr/bin/python3 /path/to/your/orchestrate_r_job.py --input_file /data/daily_sensor_readings.csv --output_file /results/daily_stats.csv --r_log_file /tmp/r_daily_run.log --analysis_type t_test > /tmp/cron_output.log 2>&1

# Let's break down a more robust example:
# Run every day at 2 AM
# Ensure correct Python interpreter path, e.g., using `which python3`
# Ensure Rscript is in the PATH or specify its absolute path if Python calls it directly.
# It's often best practice to set the PATH within the cron job itself or use absolute paths for executables.

# Set environment variables for cron job for robust execution
# Example: Setting PATH for Python and Rscript if they are not in cron's default PATH
PATH=/usr/local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/sbin:/opt/R/4.2.0/bin:/opt/conda/bin

# Daily T-test analysis at 02:00 AM
0 2 * * * /usr/bin/python3 /path/to/your/orchestrate_r_job.py --input_file /data/daily_sensor_readings.csv --output_file /results/daily_stats.csv --r_log_file /tmp/r_daily_run.log --analysis_type t_test > /var/log/my_bio_pipeline/daily_run_stdout.log 2>&1

# Weekly Linear Model analysis every Monday at 04:30 AM
30 4 * * 1 /usr/bin/python3 /path/to/your/orchestrate_r_job.py --input_file /data/weekly_experiment_data.csv --output_file /results/weekly_lm_results.csv --r_log_file /tmp/r_weekly_run.log --analysis_type linear_model > /var/log/my_bio_pipeline/weekly_run_stdout.log 2>&1

# Important considerations:
# - Replace `/path/to/your/` with actual absolute paths.
# - Ensure the log directories (`/var/log/my_bio_pipeline/` and `/tmp/`) exist and are writable.
# - `> /path/to/stdout.log 2>&1` redirects both standard output and standard error to a log file.
#   This is critical for debugging cron jobs, as cron does not interactively display output.
# - Test your cron job with a simple command first, e.g., `* * * * * echo "Hello from cron" >> /tmp/cron_test.log`

#### Example: Python for Email Notification on Failure

# Add this function to your orchestrate_r_job.py script
# Make sure to install smtplib and email.mime.text if not available: 
# pip install smtplib email

import smtplib
from email.mime.text import MIMEText

def send_email_notification(subject, body, sender_email, receiver_email, smtp_server, smtp_port, smtp_username, smtp_password):
    msg = MIMEText(body)
    msg['Subject'] = subject
    msg['From'] = sender_email
    msg['To'] = receiver_email

    try:
        with smtplib.SMTP_SSL(smtp_server, smtp_port) as server: # Use SMTP_SSL for secure connection
            server.login(smtp_username, smtp_password)
            server.send_message(msg)
        log_message(f"Email notification sent successfully to {receiver_email}")
    except Exception as e:
        log_message(f"Failed to send email notification: {e}", "ERROR")

# --- Integrate into your main() function after error blocks ---
# Example usage in orchestrate_r_job.py (after a CalledProcessError or other exception):
# ... inside the except subprocess.CalledProcessError as e: block ...
#    error_body = f"R script execution failed.\nDetails:\n{e.stderr}\nR Log: {args.r_log_file}"
#    send_email_notification(
#        "CRITICAL: Bio-Pipeline R Job Failed", 
#        error_body, 
#        "your_sender_email@example.com", 
#        "your_receiver_email@example.com", 
#        "smtp.example.com", 
#        465, # or 587 for STARTTLS
#        "smtp_username", 
#        "smtp_password"
#    )
#    sys.exit(1)


#### Example: Simple shell script to wrap cron job for additional logging/error handling

# File: run_pipeline.sh
#!/bin/bash

LOG_DIR="/var/log/my_bio_pipeline"
mkdir -p "$LOG_DIR"
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
STDOUT_LOG="$LOG_DIR/pipeline_stdout_${TIMESTAMP}.log"
STDERR_LOG="$LOG_DIR/pipeline_stderr_${TIMESTAMP}.log"

# Full path to your Python orchestrator script
PYTHON_ORCHESTRATOR="/path/to/your/orchestrate_r_job.py"

# Arguments for the Python script - adjust as needed
INPUT_DATA="/data/sensor_data_$(date +%Y%m%d).csv"
OUTPUT_RESULTS="/results/analysis_${TIMESTAMP}.csv"
R_RUN_LOG="/tmp/r_script_run_${TIMESTAMP}.log"
ANALYSIS_TYPE="linear_model"

# Execute the Python script, redirecting its stdout/stderr to separate files
/usr/bin/python3 "$PYTHON_ORCHESTRATOR" \
    --input_file "$INPUT_DATA" \
    --output_file "$OUTPUT_RESULTS" \
    --r_log_file "$R_RUN_LOG" \
    --analysis_type "$ANALYSIS_TYPE" > "$STDOUT_LOG" 2> "$STDERR_LOG"

EXIT_CODE=$?

if [ $EXIT_CODE -ne 0 ]; then
    echo "Pipeline failed with exit code $EXIT_CODE. See $STDERR_LOG for details." | mail -s "CRITICAL: Bio-Pipeline Failure" your_email@example.com
else
    echo "Pipeline completed successfully. See $STDOUT_LOG for details." | mail -s "SUCCESS: Bio-Pipeline Completion" your_email@example.com
fi

# Update crontab to call this shell script instead:
# 0 3 * * * /path/to/your/run_pipeline.sh

#### Example: Using a dedicated user for cron jobs

# 1. Create a new user with restricted permissions (e.g., 'bio_pipeline_user')
# sudo adduser bio_pipeline_user

# 2. Switch to this user to set up their crontab
# sudo su - bio_pipeline_user
# crontab -e

# 3. Add your cron entries here. Example:
# 0 2 * * * /usr/bin/python3 /home/bio_pipeline_user/scripts/orchestrate_r_job.py --input_file /data/raw/daily.csv --output_file /data/processed/daily_stats.csv --r_log_file /home/bio_pipeline_user/logs/r_daily_run.log > /home/bio_pipeline_user/logs/cron_daily_stdout.log 2>&1

# 4. Ensure necessary file permissions:
# The 'bio_pipeline_user' must have read access to input data directories.
# The 'bio_pipeline_user' must have write access to output data directories and log directories.
# Example: Setting permissions for data directories (adjust group as needed)
# sudo chown -R bio_pipeline_user:bio_data_group /data/raw
# sudo chmod -R 750 /data/raw
# sudo chown -R bio_pipeline_user:bio_data_group /data/processed
# sudo chmod -R 770 /data/processed # If processed data needs to be written

#### Example: Version Control for Scripts

# Ensure all R and Python scripts are managed under Git or similar VCS.
# This allows tracking changes, collaboration, and easy rollback.

# Typical workflow:
# git init
# git add statistical_analysis.R orchestrate_r_job.py
# git commit -m "Initial commit of R and Python pipeline scripts"
# git branch -M main
# git remote add origin git@github.com:yourorg/your-bio-repo.git # Replace with your repo
# git push -u origin main

# When updating scripts:
# git pull origin main
# Make changes...
# git add .
# git commit -m "Improved error logging in Python orchestrator"
# git push origin main

# Ensure the cron user pulls the latest scripts or scripts are deployed from VCS.