tokenize.c

#include "mysh.h"

/*
 * tokenize.c  — Phase 1 (Varnika)
 *
 * Turns a raw input string into a null-terminated array of token strings.
 * Handles:
 *   - whitespace delimiters (space, tab)
 *   - single-quoted strings  ('hello world' → one token, quotes stripped)
 *   - double-quoted strings  ("hello world" → one token, quotes stripped)
 *   - operators treated as their own tokens: | > < >> &
 *   - trailing newline stripped
 *   - empty / comment lines   (returns NULL)
 *
 * Returns a heap-allocated array of heap-allocated strings.
 * Caller must free with free_tokens().
 */

/* ── helpers ──────────────────────────────────────────── */

static int is_operator_char(char c) {
    return c == '|' || c == '>' || c == '<' || c == '&';
}

/* ── tokenize ─────────────────────────────────────────── */

char **tokenize(char *input) {
    if (!input) return NULL;

    /* strip trailing newline / whitespace */
    int len = strlen(input);
    while (len > 0 && (input[len-1] == '\n' || input[len-1] == '\r' || input[len-1] == ' '))
        input[--len] = '\0';

    /* skip empty lines and comments */
    int i = 0;
    while (input[i] == ' ' || input[i] == '\t') i++;
    if (input[i] == '\0' || input[i] == '#')
        return NULL;

    char **tokens = malloc(MAX_TOKENS * sizeof(char *));
    if (!tokens) { perror("malloc"); exit(1); }

    int tok_count = 0;
    char buf[MAX_INPUT];

    while (input[i] != '\0' && tok_count < MAX_TOKENS - 1) {

        /* skip whitespace */
        if (input[i] == ' ' || input[i] == '\t') { i++; continue; }

        /* single-quoted string: 'hello world' */
        if (input[i] == '\'') {
            i++;  /* skip opening quote */
            int j = 0;
            while (input[i] != '\'' && input[i] != '\0')
                buf[j++] = input[i++];
            buf[j] = '\0';
            if (input[i] == '\'') i++; /* skip closing quote */
            tokens[tok_count++] = strdup(buf);
            continue;
        }

        /* double-quoted string: "hello world" */
        if (input[i] == '"') {
            i++;  /* skip opening quote */
            int j = 0;
            while (input[i] != '"' && input[i] != '\0')
                buf[j++] = input[i++];
            buf[j] = '\0';
            if (input[i] == '"') i++; /* skip closing quote */
            tokens[tok_count++] = strdup(buf);
            continue;
        }

        /* >> must be detected before > */
        if (input[i] == '>' && input[i+1] == '>') {
            tokens[tok_count++] = strdup(">>");
            i += 2;
            continue;
        }

        /* single-character operators: | > < & */
        if (is_operator_char(input[i])) {
            buf[0] = input[i++];
            buf[1] = '\0';
            tokens[tok_count++] = strdup(buf);
            continue;
        }

        /* regular word token — read until whitespace or operator */
        int j = 0;
        while (input[i] != '\0' &&
               input[i] != ' '  && input[i] != '\t' &&
               input[i] != '\'' && input[i] != '"'  &&
               !is_operator_char(input[i])) {
            buf[j++] = input[i++];
        }
        buf[j] = '\0';
        if (j > 0)
            tokens[tok_count++] = strdup(buf);
    }

    tokens[tok_count] = NULL; /* null-terminate */
    return tokens;
}

/* ── free_tokens ──────────────────────────────────────── */

void free_tokens(char **tokens) {
    if (!tokens) return;
    for (int i = 0; tokens[i]; i++)
        free(tokens[i]);
    free(tokens);
}

/*
 * parse.c (same file for convenience)
 *
 * Turns token array into an array of Command structs.
 * Splits on '|'.  Detects '<', '>', '>>',  '&'.
 * Calls Anusha's expand_vars() on each argv token.
 *
 * Returns heap-allocated Command array; *n_cmds set to count.
 * Caller must free with free_commands().
 */

Command *parse(char **tokens, int *n_cmds) {
    *n_cmds = 0;
    if (!tokens || !tokens[0]) return NULL;

    Command *cmds = malloc(MAX_COMMANDS * sizeof(Command));
    if (!cmds) { perror("malloc"); exit(1); }

    /* zero everything out */
    memset(cmds, 0, MAX_COMMANDS * sizeof(Command));

    int cmd_idx   = 0;
    int argc      = 0;
    char **argv   = malloc(MAX_TOKENS * sizeof(char *));
    if (!argv) { perror("malloc"); exit(1); }

    /* check if last real token is '&' */
    int last = 0;
    while (tokens[last]) last++;
    int global_bg = 0;
    if (last > 0 && strcmp(tokens[last-1], "&") == 0) {
        global_bg = 1;
        tokens[last-1] = NULL; /* remove & from token list */
    }

    for (int i = 0; tokens[i] != NULL; i++) {

        /* pipe: finish current command, start new one */
        if (strcmp(tokens[i], "|") == 0) {
            argv[argc] = NULL;
            cmds[cmd_idx].argv = argv;
            cmds[cmd_idx].background = 0; /* only last cmd gets bg flag */
            cmd_idx++;
            argc = 0;
            argv = malloc(MAX_TOKENS * sizeof(char *));
            if (!argv) { perror("malloc"); exit(1); }
            continue;
        }

        /* input redirect: < filename */
        if (strcmp(tokens[i], "<") == 0) {
            if (tokens[i+1]) {
                cmds[cmd_idx].infile = strdup(tokens[++i]);
            } else {
                fprintf(stderr, "mysh: expected filename after <\n");
            }
            continue;
        }

        /* append redirect: >> filename */
        if (strcmp(tokens[i], ">>") == 0) {
            if (tokens[i+1]) {
                cmds[cmd_idx].outfile = strdup(tokens[++i]);
                cmds[cmd_idx].append  = 1;
            } else {
                fprintf(stderr, "mysh: expected filename after >>\n");
            }
            continue;
        }

        /* output redirect: > filename */
        if (strcmp(tokens[i], ">") == 0) {
            if (tokens[i+1]) {
                cmds[cmd_idx].outfile = strdup(tokens[++i]);
                cmds[cmd_idx].append  = 0;
            } else {
                fprintf(stderr, "mysh: expected filename after >\n");
            }
            continue;
        }

        /* regular argument — expand $VAR first (Anusha's function) */
        argv[argc++] = expand_vars(tokens[i]);
    }

    /* finish last command */
    argv[argc] = NULL;
    cmds[cmd_idx].argv       = argv;
    cmds[cmd_idx].background = global_bg;
    cmd_idx++;

    *n_cmds = cmd_idx;
    return cmds;
}

/* ── free_commands ────────────────────────────────────── */

void free_commands(Command *cmds, int n) {
    if (!cmds) return;
    for (int i = 0; i < n; i++) {
        if (cmds[i].argv) {
            for (int j = 0; cmds[i].argv[j]; j++)
                free(cmds[i].argv[j]);
            free(cmds[i].argv);
        }
        if (cmds[i].infile)  free(cmds[i].infile);
        if (cmds[i].outfile) free(cmds[i].outfile);
    }
    free(cmds);
}