Crawler backend

sitescripts/crawler/schema.sql

 DROP TABLE IF EXISTS crawler_sites;
 DROP TABLE IF EXISTS crawler_runs;
 DROP TABLE IF EXISTS crawler_data;
 
 CREATE TABLE crawler_sites (
        id INT NOT NULL AUTO_INCREMENT,
        PRIMARY KEY (id),
-       url VARCHAR(255) NOT NULL
+       url VARCHAR(512) NOT NULL

[Wladimir Palant, 2012/09/14 17:24:18]

How about UNIQUE(url) constraint? This will make sure we don't add the same URL
twice due to some bug. And it will create an index (which is necessary given
that you frequently query this table by URL).

[Felix Dahlke, 2012/09/26 15:20:30]

Done.

 );
 
 CREATE TABLE crawler_runs (
        id INT NOT NULL AUTO_INCREMENT,
        PRIMARY KEY (id),
        timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP
 );
 
 CREATE TABLE crawler_data (
        id INT NOT NULL AUTO_INCREMENT,
        PRIMARY KEY (id),
        run INT NOT NULL,
        FOREIGN KEY (run) REFERENCES crawler_runs (id),
        site INT NOT NULL,
        FOREIGN KEY (site) REFERENCES crawler_sites (id),
-       request_url VARCHAR(255) NOT NULL,
-       document_url VARCHAR(255) NOT NULL
+       url VARCHAR(512) NOT NULL,
+       filtered BOOLEAN NOT NULL
 );