#include <asvtools.h>

#include <string.h>
#include <stdlib.h>
#include <ctype.h>

typedef struct
{
    int id;
    char *n, *sect, *keywords;
}
coin_t;

int    n_coins;
coin_t *coins;

/* ---------------------------------------------------------------- */

void dehtml (char *s)
{
    char *p;
    
    for (p = s; *p && *(p+1) && *(p+2) && *(p+3) && *(p+4) && *(p+5); p++)
    {
        if (*p == '&' &&
            *(p+1) == '#' &&
            isdigit ((unsigned)(*(p+2))) &&
            isdigit ((unsigned)(*(p+3))) &&
            isdigit ((unsigned)(*(p+4))) &&
            *(p+5) == ';'
           )
        {
            *p = (p[2]-'0')*100 + (p[3]-'0')*10 + p[4]-'0';
            str_delete (s, p+1);
            str_delete (s, p+1);
            str_delete (s, p+1);
            str_delete (s, p+1);
            str_delete (s, p+1);
        }
    }
}

/* ---------------------------------------------------------------- */

#define LOAD_DEBUG FALSE

int load_coins (void)
{
    int  i, nc, nw;
    char **lines, **w;
    static FILE *fp=NULL;

    if (LOAD_DEBUG && fp == NULL) fp = xfopen ("load.debug", "w");

    nc = load_textfile ("keywords.txt", &lines);
    if (nc <= 0) return -1;

    n_coins = 0;
    coins = xmalloc (sizeof (coin_t) * nc);
    for (i=0; i<nc; i++)
    {
        nw = str_split (lines[i], &w, '\t');
        if (nw != 4)
        {
            warning1 ("line starting with %s: not 4 fields\n", w[0]);
            continue;
        }

        if (strcmp (w[0], "id") == 0) continue;

        if (w[3][0] == '\0') continue; /* empty description */

        coins[n_coins].id = atoi (w[0]);
        coins[n_coins].sect = w[1];
        coins[n_coins].n = w[2];
        dehtml (w[3]);
        coins[n_coins].keywords = w[3];
        n_coins++;

        free (w);
    }

    free (lines);

    if (fp != NULL)
    {
        for (i=0; i<n_coins; i++)
            fprintf (fp, "%d,%s\n", coins[i].id, coins[i].keywords);
        xfclose (fp);
    }

    return 0;
}


/* ---------------------------------------------------------------- */

typedef struct
{
    char *word;

    int n, n_a;
    int *ids;
}
word_t;

int n_words, n_words_a;
word_t *words;

/* ---------------------------------------------------------------- */

void add_keywords (int id, int n_keywords, char **keywords)
{
    int i, j;

    for (i=0; i<n_keywords; i++)
    {
        /* find keyword in the index. add if absent */
        for (j=0; j<n_words; j++)
            if (strcmp (words[j].word, keywords[i]) == 0) break;

        if (j == n_words)
        {
            if (n_words == n_words_a)
            {
                n_words_a *= 2;
                words = xrealloc (words, sizeof (word_t) * n_words_a);
            }

            words[n_words].word = strdup (keywords[i]);
            words[n_words].n_a = 8;
            words[n_words].n = 0;
            words[n_words].ids = xmalloc (sizeof (int) * words[n_words].n_a);
            n_words++;
        }

        /* insert id into the id list */
        if (words[j].n_a == words[j].n)
        {
            words[j].n_a *= 2;
            words[j].ids = xrealloc (words[j].ids, sizeof (int) * words[j].n_a);
        }

        words[j].ids[words[j].n++] = id;
    }
}

/* ---------------------------------------------------------------- */

#define PARSE_DEBUG FALSE

int parse_keywords (char *s, char **keywords)
{
    int i, j, nw, nw1, nk;
    char *p, **w, **w1;
    static FILE *fp=NULL;

    if (PARSE_DEBUG && fp == NULL) fp = xfopen ("parse.debug", "w");

    p = strdup (s);
    nw = str_split (p, &w, ',');

    nk = 0;
    /*for (i=0; i<nw; i++)
    {
        keywords[nk++] = strdup (w[i]);
        str_strip2 (keywords[i], " \r\n\t");
    }*/

    /* now we need to split each word into index tokens */
    for (i=0; i<nw; i++)
    {
        for (j=0; w[i][j] != '\0'; j++)
        {
            if (w[i][j] >= 'a' && w[i][j] <= 'z') continue;
            if (w[i][j] >= '0' && w[i][j] <= '9') continue;
            if (w[i][j] >= 'A' && w[i][j] <= 'Z') w[i][j] += 'a'-'A';
            if (w[i][j] == '\'') continue;
            if (w[i][j] == '_') continue;
            w[i][j] =  ' ';
        }

        nw1 = str_words (w[i], &w1, WSEP_SPACES);
        for (j=0; j<nw1; j++)
            keywords[nk++] = strdup (w1[j]);
        free (w1);
    }

    if (fp != NULL)
    {
        fprintf (fp, "%s ->\n", s);
        for (i=0; i<nw; i++)
            fprintf (fp, "  %d. %s\n", i+1, keywords[i]);
        fprintf (fp, "\n");
    }

    free (w);
    return nk;
}


/* ---------------------------------------------------------------- */

static int cmp_word (const void *e1, const void *e2)
{
    word_t *w1, *w2;

    w1 = (word_t *)e1;
    w2 = (word_t *)e2;

    return strcmp (w1->word, w2->word);
}

/* ---------------------------------------------------------------- */

int write_index (void)
{
    int i, j;
    FILE *fp;

    qsort (words, n_words, sizeof (word_t), cmp_word);

    fp = xfopen ("keyword_index.sql", "w");

    fprintf (fp, "TRUNCATE `h001273440_coins`.`tab_index`;\n");

    for (i=0; i<n_words; i++)
    {
        qsort (words[i].ids, words[i].n, sizeof (int), cmp_integers);
        words[i].n = uniq (words[i].ids, words[i].n, sizeof (int), cmp_integers);

        /* fprintf (fp, "\"%s\";\"", words[i].word); */
        if (strchr (words[i].word, '\'') != NULL)
        {
            char *p = str_strdup1 (words[i].word, 10);
            str_replace (p, "'", "\\'", FALSE);
            fprintf (fp, "INSERT INTO `tab_index` (`keyword`, `ind`) VALUES ('%s', '", p);
            free (p);
        }
        else
            fprintf (fp, "INSERT INTO `tab_index` (`keyword`, `ind`) VALUES ('%s', '", words[i].word);
        for (j=0; j<words[i].n; j++)
        {
            fprintf (fp, "%d", words[i].ids[j]);
            if (j != words[i].n-1) fprintf (fp, ",");
        }
        /*fprintf (fp, "\"\n");*/
        fprintf (fp, "');\n");
    }

    xfclose (fp);
    return 0;
}

/* ---------------------------------------------------------------- */

int main (int argc, char *argv[])
{
    int rc, i, n;
    char *keywords[1024];
    double t1;

    t1 = clock1 ();
    /* read file keywords.txt and parse it */
    rc = load_coins ();
    if (rc < 0) return -1;
    warning1 ("%d coins loaded\n", n_coins);

    /* initialize empty index */
    n_words_a = 1024;
    n_words = 0;
    words = xmalloc (sizeof (word_t) * n_words_a);

    /* parse keywords, insert them into index */
    for (i=0; i<n_coins; i++)
    {
        n = parse_keywords (coins[i].keywords, keywords);
        add_keywords (coins[i].id, n, keywords);
    }

    write_index ();

    warning1 ("done in %.3f sec\n", clock1()-t1);

    return 0;
}