package relational.email;

import proof.*;
import sdsi.*;
import ssh.*;
import relational.*;
import rmi.*;
import Tools.*;

import java.util.*;
import java.io.*;
import java.net.*;
import java.lang.reflect.*;
import java.rmi.*;

/**
 * A tool for importing mail from a Berkeley-style mail folder into a
 * relational email database.
 */
public class Mailbox {

	static class PendingMailbox {
		Vector mv, bv, hv;

		PendingMailbox() {
			mv = new Vector();
			bv = new Vector();
			hv = new Vector();
		}

		void add(PendingMessage pm) {
			mv.add(pm.msg);
			bv.add(pm.body);
			hv.addAll(pm.hdrs);
		}

		Vector insertAll(Database db) {
			Iterator iter;
/*
			int count=0;
*/

			// Send bodies one at a time (64k limit on RMI calls -- blech)
			// Not sure how to handle mail (gasp) *longer* than 64K!
			// Perhaps break body into pieces? Ugh.
			iter = bv.iterator();
			while (iter.hasNext()) {
				try {
					((Relational) iter.next()).insert();
				} catch (RemoteException ex) {
					System.out.println("Insert object failed: "+ex.toString());
				}
			}
				
/*
			// count body data bytes
			iter = bv.iterator();
			while (iter.hasNext()) {
				Body b = (Body) iter.next();
				count += b.body.length();
			}
			System.out.println("Body bytes: "+count);
*/
			
			// send msgs in blocks of 1000
			System.out.println("There are "+mv.size()+" messages.");
			Relational[] mra = (Relational[])
				mv.toArray(new Relational[mv.size()]);
			blockInsert(db, mra, 1000);

			// send headers in blocks of 100
			// TODO (could still have an overrun. that would suck. slurp.)
			System.out.println("There are "+hv.size()+" headers.");
			Relational[] hra = (Relational[])
				hv.toArray(new Relational[hv.size()]);
			blockInsert(db, hra, 100);

/*
			// count header data bytes
			sdv = new Vector();
			count=0;
			iter = hv.iterator();
			while (iter.hasNext()) {
				Header h = (Header) iter.next();
				// count += h.name.length();
				// count += h.value.length();
				// save only bytes; see how much memory appears
				byte[] ca = h.name.getBytes();
				sdv.add(ca);
				count += ca.length;
				ca = h.value.getBytes();
				sdv.add(ca);
				count += ca.length;
			}
			System.out.println("Header bytes: "+count);
*/

			return mv;
		}
		static Object x;

		void blockInsert(Database db, Relational[] ary, int blockLength) {
			try {
				Relational[] block = null;
				for (int off=0; off<ary.length; off+=blockLength) {
					int len = Math.min(blockLength, ary.length-off);
					if (len<blockLength) {
						// shorten block ary for last group
						block = new Relational[len];
					} else if (block==null) {
						// don't make the long array if ary.size()<blockLength
						block = new Relational[blockLength];
					}
					System.arraycopy(ary, off, block, 0, len);
					db.insert(block);
				}
			} catch (RemoteException ex) {
				comment(2, ex.toString());
			}
		}
	}

	static class PendingMessage {
		Database db;
		Vector hdrs;
		Body body;
		StringBuffer bodysb;
		Message msg;
		boolean valid;

		PendingMessage(Database db) {
			this.db = db;
			valid = false;
			bodysb = new StringBuffer();
			hdrs = new Vector();
			msg = new Message(db);
			body = new Body(db);
			body.setMsg(msg);
		}

		void nextHeader(String n, String s, String v, boolean syn) {
			Header h = new Header(db);
			h.name = n;
			h.whitespace = s;
			h.value = v;
			// h.order = hdrs.size();
			h.synthetic = syn;
			h.setMsg(msg);
			hdrs.add(h);
			valid = true;
		}

		void headerContinuation(String v2) {
			if (hdrs.size()<=0) {
				throw new RuntimeException("Imported mailbox corrupt: "
					+"header continuation without header");
			}
			Header h = (Header) hdrs.elementAt(hdrs.size()-1);
			h.value += "\n"+v2;
		}

		void nextMessageLine(String s) {
			if (hdrs.size()<=0) {
				// message never started
				throw new RuntimeException("Imported mailbox corrupt: "
					+"does not start with Unix From header");
			}
			bodysb.append(s);
			bodysb.append('\n');
		}

		void complete(PendingMailbox pmbx) {
			if (!valid) {
				// PendingMessage was created but never filled in.
				// Don't create an empty message.
				return;
			}

			// sort synthetic headers after real headers
			Collections.sort(hdrs, SyntheticComparator.sc);
			// now assign header order numbers in their sorted order
			int order=0;
			Iterator iter = hdrs.iterator();
			while (iter.hasNext()) {
				Header h = (Header) iter.next();
				h.order = order++;
			}

			body.body = bodysb.toString();
			pmbx.add(this);
		}
	}

	static class SyntheticComparator
		implements Comparator {
		public int compare(Object o1, Object o2) {
			Header h1 = (Header) o1;
			Header h2 = (Header) o2;
			if (!h1.synthetic && h2.synthetic) {
				return -1;
			} else if (h1.synthetic && !h2.synthetic) {
				return 1;
			} else {
				return 0; // don't perturb order; exploit stable sort
			}
		}
		public static SyntheticComparator sc = new SyntheticComparator();
	}

	// 2-node state machine; diagram on journal page 9/20/1999.3
	static int SKIP_ONE_HEADER = 12;
	static int HEADERS = 13;
	static int BODY = 14;

	/**
	 * Parse mail from InputStream <code>is</code> into database
	 * <code>db</code>. The original <code>folderName</code> is
	 * attached to each message as a synthetic header to preserve the
	 * user's categorization.
	 */
	public static Vector importMail(Database db, InputStream is,
		String folderName)
		throws RemoteException {

		ProgressBar pb;
		int filesize = 1;
		try {
			filesize = is.available();
		} catch (IOException ex) {}
		pb = new ProgressBar(filesize);

		PendingMailbox pmbx = new PendingMailbox();

		int curState = BODY;
		int nextState = BODY;

		PendingMessage pm = new PendingMessage(db);

		BufferedReader br = new BufferedReader(new InputStreamReader(is));
		try {
			int count=0;
			System.out.println("Importing messages:");
			while (true) {
				String s = br.readLine();
				if (s==null) {	// inconsistent ways of getting EOF
					break;
				}

				// Figure out which state is next
				if (curState == BODY) {
					if (s.startsWith("From ")) {
						// new message starts here
						pm.complete(pmbx);
							// write out previous message, if any
						pm = new PendingMessage(db);

						// the action of creating msg & body occur upon
						// leaving a state, because they occur
						// exactly on the edge from BODY to HEADERS

						++count;
//						if (count%10==0) {
//							System.err.print(".");
//							System.err.flush();
//						}

						pm.nextHeader("X-Folder", " ",
							folderName, true);
						pm.nextHeader("X-Folder-Order", " ",
							String.valueOf(count), true);
						pm.nextHeader("X-BSD-From", " ",
							s.substring(5), true);

						nextState = SKIP_ONE_HEADER;
							// skip over the X-BSD-From header, now.

						// update the progress bar
						int remaining=filesize;
						try {
							remaining = is.available();
						} catch (IOException ex) {}
						pb.update(filesize-remaining);
					} else {
						nextState = BODY;
					}
				} else {
					// HEADERS
					if (s.equals("")) {
						nextState = BODY;
					}
				}

				// actions that occur upon entering a state
				if (nextState == BODY) {
					pm.nextMessageLine(s);
				} else if (nextState == HEADERS) {
					// HEADERS
					int colonIndex;
					if (s.charAt(0)==' ' || s.charAt(0)=='\t') {
						pm.headerContinuation(s);
					} else if ((colonIndex = s.indexOf(':'))>=0) {
						String headerValue = s.substring(colonIndex+1);
						// strip prefix spaces
						int si=0;
						while (si<headerValue.length()
							&& (headerValue.charAt(si)==' '
								|| headerValue.charAt(si)=='\t')) {
								si++;
						}
						pm.nextHeader(s.substring(0, colonIndex),
							headerValue.substring(0, si),
							headerValue.substring(si), false);
					} else {
						pm.nextHeader("X-Malformed", "", s, true);
					}
				} else if (nextState == SKIP_ONE_HEADER) {
					nextState = HEADERS;
				} else {
					Tools.Assert.assert(false, "unknown state");
				}
				curState = nextState;
			}
			// archive is exhausted -- write out final message
			pm.complete(pmbx);
		} catch (IOException ex) {
			comment(1, "Done processing stream (IOException)");
			pm.complete(pmbx);
		}

		// finish the progress bar
		pb.done();
		return pmbx.insertAll(db);
	}

	/**
	 * Import mail given a Unix filename.
	 */
	public static Vector importMail(Database db, String filename)
		throws FileNotFoundException, RemoteException {
		FileInputStream fis = null;
		fis = new FileInputStream(filename);

		return importMail(db, fis, filename);
	}

	/**
	 * Debug tool.
	 */
	public static void comment(int i, String s) {
		if (i==2) {
			System.out.println(s);
		}
	}

	/**
	 * Unix command-line interface to the mail import tool.
	 */
	public static void main(String argv[]) {
		try {
			Memory mt;
			Timer t;

			SSHContext myContext;

			// code block stolen from MailServlet.java
			try {
				// Save time when experimenting by loading up a stale
				// old keypair for SSH communication. In real life,
				// you'd use fresh keys, or at least get the key from
				// a crypto agent program responsible for keeping keys fresh.
				String cheaterPriv = "certs-sharedserver/id.private";
				String cheaterPub = "certs-sharedserver/id.public";
				SDSIRSAPrivateKey priv =
					(SDSIRSAPrivateKey) KeyTools.processFilename(cheaterPriv);
				SDSIRSAPublicKey pub =
					(SDSIRSAPublicKey) KeyTools.processFilename(cheaterPub);
				myContext = new SSHContext(ssh.RSA.RSAKey.fromRSAPrivateKey(priv),
					ssh.RSA.RSAKey.fromRSAPublicKey(pub));
				System.err.println("cheated by loading identity from file "+cheaterPriv);
			} catch (Exception ex) {
				System.err.println("creating a keypair for this service."
					+" (cheating failed with "+ex.toString()+")");
				myContext = SSHContext.newKeys();
				System.err.println("done creating keypair.");
			}
			Prover2 prover = new Prover2("certs-jon");
			SDSIKeyPair skp = new SDSIKeyPair(myContext.getPrivateKey(),
					myContext.getPublicKey());
			prover.introduceObject(skp);
			prover.loadCache();
			InvokeHack.setCurrentProver(prover);

			Database db;
			if (argv.length>=2 && argv[1].equals("remote")) {
				InetAddress thisHost = InetAddress.getLocalHost();
				db = (Database)
					Naming.lookup("//"+thisHost.getHostName()+"/RMIDatabase");
			} else {
				db = new InternalDatabase();
			}

			String filename = argv[0];

			mt = new Memory();
			t = new Timer();
			Vector v = importMail(db, filename);
			comment(2, "import phase: "+t.wallTime()+" sec; "+mt);

			mt = new Memory();
			t = new Timer();
			// comment(2, "indexing ---------------");
			db.createIndex(Header.f_msg);
			db.createIndex(Header.f_value);
			comment(2, "index phase: "+t.wallTime()+" sec; "+mt);

			// comment(2, "---------------");
			t = new Timer();
			// Build internal Select statement.
			FromClause ifc = FromClause.createAnonymous(Header.class);
			ColumnSpec ics = ColumnSpec.create(
				ifc,
				new int[] { 0 },
				new FieldDescriptor[] { Header.f_msg });
			Select isel = new Select(ics,
							Where.equals(Header.f_name, "Subject")
						);

			Header dummyHeader = new Header(db);
			dummyHeader.name = "Subject";
			dummyHeader.value = "[missing]";
			dummyHeader.synthetic = true;
			dummyHeader.order = 0;	// unknown
			dummyHeader.msg_fk = new Double(0.0);	// matches no Message

			// Build outside select statement
//			FromClause ofc = FromClause.createAnonymous(Message.class);
			FromClause ofc = FromClause.create(
				new String[] { "Message", "Header" },
				new Class[] { Message.class, Header.class });
			ColumnSpec ocs = ofc.getNaturalColumnSpec();
			Select osel = new Select(ocs, ofc,
				new WhereOr(
					new WhereAnd(
						new WhereIn(Message.f_primaryKey, isel),
						new WhereAnd(
							new WhereEquals(Header.f_name, "Subject"),
							new WhereJoin(Message.f_primaryKey, Header.f_msg)
						)
					),
					new WhereAnd(
						new WhereNot(
							new WhereIn(Message.f_primaryKey, isel)
						),
						new WhereConstant("Header", dummyHeader)
					)
				)
			);
			osel.setDistinct(true);
			osel.setOrderBy(new OrderByOne(Header.f_value,
				new SubjectComparator()));

			System.exit(0);	 //tmp debug TODO delete
			ResultSet rs = db.evaluateSelect(osel);
			comment(2, "select phase: "+t.wallTime());

			t = new Timer();
			v = rs.getVector();
			ColumnSpec rcs = rs.getColumnSpec();
			int msgFieldIndex = rcs.findField(Message.f_reference);
			int hdrFieldIndex = rcs.findField(Header.f_reference);
			System.out.println(v.size()+" messages:");
			for (int i=0; i<v.size(); i++) {
				Header h = (Header) rcs.getField(
					(Row) v.elementAt(i), hdrFieldIndex);
				System.err.println("subj: "+h.value);

/*
				Message m = (Message) rcs.getField(
					(Row) v.elementAt(i), msgFieldIndex);

				Select sel = new Select(Header.class,
					Where.and(
						Where.equals(Header.f_msg, m.primaryKey),
						Where.equals(Header.f_name, "Subject")
					)
				);
				sel.setOrderBy(OrderByOne.natural(Header.f_order));
				Vector hv = sel.evaluate(db).getVector();
					
				for (int j=0; j<hv.size(); j++) {
					Header h = (Header) hv.elementAt(j);
					System.err.println(m.primaryKey+"*"+h.name+": "+h.value);
				}
*/
			}
			comment(2, "display phase: "+t.wallTime());
		} catch (Exception ex2) {
			ex2.printStackTrace();
		}
	}
}
