package relational;

import Tools.*;
import ssl.*;
import ssh.*;
import rmi.*;
import sdsi.*;
import proof.*;
import sdsi.sexp.*;
import COM.claymoresystems.ptls.*;

import java.lang.reflect.Field;
import java.lang.reflect.Constructor;
import java.util.*;
import java.rmi.RemoteException;
import java.rmi.server.UnicastRemoteObject;
import java.text.NumberFormat;

/**
 * My implementation of the Database interface. Supports select
 * statements and indexing.
 *
 * @todo An implementation of Relational needs a way to always be able to
 * invert any pointer. One really crummy mechanism is to keep track of
 * all Relationals of each type, and when asking for the pointers from
 * a given type, iterate through the list of existing guys.
 * 
 * @todo Some small issues with references and never garbage collecting are
 * sure to show up.
 * 
 * @todo subclasses of Relational classes don't work yet.
 */
public class InternalDatabase
	extends UnicastRemoteObject
	implements Database {

	static final boolean testingIndex = false;

	public InternalDatabase()
		throws RemoteException {
	}

	/**
	 * Create an InternalDatabase object that is accessed via RMI-over-SSL.
	 * [I couldn't get RMI-over-SSL working reliably, so I switched to my
	 * SSH implementation.]
	 *
	 * @param context The SSLContext object to use with the SSL connection.
	 */
	public InternalDatabase(SSLContext context)
		throws RemoteException {
		super(0 /*port*/,
			new SSLClientSocketFactory(context),
			new SSLServerSocketFactory(context));
	}

	/**
	 * Create an InternalDatabase object that is accessed via
	 * RMI-over-SSH.
	 * 
	 * @param context The SSHContext object to use with the SSH
	 * connection. 
	 */
	public InternalDatabase(SSHContext context, SDSIPrincipal serverIssuer)
		throws RemoteException {
		super(0 /*port*/,
			new SSHClientSocketFactory(context),
			new SSHServerSocketFactory(context));
		initAuthorization(serverIssuer);
	}

	Map tableRowsByClass = new HashMap();

	public void shutdown()
		throws RemoteException {
		checkAuth();
		System.exit(0);
	}

	public void insert(Relational ro) {
		// TODO: could imagine a faster specialized impl of the subroutine
		insert(new Relational[] {ro});
	}

	public void insert(Relational[] ros) {
		checkAuth();
		Class c = ros[0].getClass();
		HashMap objectList = (HashMap) tableRowsByClass.get(c);
		// TODO: HashSets are grossly inefficient with memory.
		if (objectList == null) {
			objectList = new HashMap();
			tableRowsByClass.put(c, objectList);
			// need to create an index for this table's primary key
			createIndex(FieldDescriptor.getPrimaryKey(c));
		}
		for (int i=0; i<ros.length; i++) {
			Tools.Assert.assert(ros[i].getClass()==c);
			Tools.Assert.assert(!objectList.containsKey(ros[i].primaryKey));
			objectList.put(ros[i].primaryKey, ros[i]);
			ros[i].valid = true;
		}

		indexSome(ros);
	}

	public void update(Relational ro) {
		// TODO: could imagine a faster specialized impl of the subroutine
		update(new Relational[] {ro});
	}

	public void update(Relational[] ros) {
		checkAuth();
		// This should be sort of equivalent to
		// UPDATE (ro.* = ...) WHERE (ro.class.PRIMARY_KEY = ro)
		// Except, of course, that if the change is made 'colocal' with
		// the database, then it takes effect (although without being
		// indexed) immediately. Creepy.
		//
		// TODO: Indices only grow; currently there's no way to remove
		// an object's membership in an index when it is updated.
		// Approach 1: remember an old version of the object, anti-index that,
		// and re-index the new version.
		// Approach 2: Periodically (on some condition) throw away the index
		// and rebuild it.
		//
		// This problem gets worse in the remote case, where every update
		// emits a garbage row (the previous holder of the unique primaryKey).

		Class rc = ros[0].getClass();

		// Replace entries in objectList with new rows
		if (Tools.Assert.getEnabled()) {
			Map objectList = (Map) tableRowsByClass.get(rc);
			for (int i=0; i<ros.length; i++) {
				Tools.Assert.assert(ros[i].getClass()==rc);
					// should already be a guy there with this key
				Relational rold =
					(Relational) objectList.get(ros[i].primaryKey);
				Tools.Assert.assert(rold!=null);
				rold.valid = false;
					// which we will replace:
				objectList.put(ros[i].primaryKey, ros[i]);
				ros[i].valid = true;
			}
		}

		indexSome(ros);
	}

	protected void indexSome(Relational[] ros)
	{
		HashSet fdSet =
			(HashSet) indexInstructions.get(ros[0].getClass());
		if (fdSet == null) {
			// no fields of this class indexed
			return;
		}

		Iterator fdIter = fdSet.iterator();
		while (fdIter.hasNext()) {
			FieldDescriptor fd = (FieldDescriptor) fdIter.next();
			Hashtable index = (Hashtable) indices.get(fd);
				// if index is null, indexInstructions was lying
			for (int i=0; i<ros.length; i++) {
				Object o = fd.get(ros[i]);
				indexOneField(index, ros[i], o);
			}
		}
	}

	/**
	 * Utility method.
	 * Finds all objects of fromClass that match the where clause.
	 * This can be used to follow pointers backwards, by using a
	 * where clause that checks for a data member that points at the
	 * target data item.
	 */
	public static Vector select(Database db, Class fromClass, Where where) {
		try {
			return db.evaluateSelect(new Select(fromClass, where)).getVector();
		} catch (RemoteException ex) {
			throw new RuntimeException(ex.toString());
		}
	}

	static int selectCount = 0;

	public ResultSet evaluateSelect(Select s) {
		// Tools.DumpProf.dump();
		timingexp.Timeline.timePoint("enter evaluateSelect");
		// Timer timer;
		// timer = new Timer();
		checkAuth();
		// System.err.println("checkAuth: "+timer);
//		timingexp.Timeline.timePoint("auth is checked");

		// timer = new Timer();
		// System.out.println("Select count: "+(++selectCount));
			// see how much RMI roundtrips are costing me
		// System.out.println("processing select:\n"+s);

		ColumnSpec cs = s.getColumnSpec();
		Where where = s.getWhere();

		// TODO worry about subclasses
		// TODO find a clever way to purge stale index info.
		// whereEquals's for indexed fields that fail should go repair the
		// index.
		ResultSet rs = boundingSuperset(where, s.getFromClause(), null);
		if (rs==null) {
			// nobody had anything to contribute -- start with all elements.
			rs = getUniverse(cs.getFromClause());
		}
		if (! rs.getFromClause().equals(s.getFromClause())) {
			// need to join in the other columns.
			rs = fillByJoin(rs, s.getFromClause());
		}

		// now verify that the result set only includes correct answers
		// (since subqueries are allowed to return extra rows if lazy.)
		// TODO: that invariant could be tightened, if the indexing were
		// made exact and all bound*() methods checked for exact
		// treatment of their input ResultSet. It would save this pass-over.
		ResultSetImpl rsi = (ResultSetImpl) rs;
		boolean copyFlag = false;
		Iterator iter = rsi.iterator();
		int count=0;
		// System.out.println("=== "+rs.hashCode()+" fc = "+rs.getFromClause());
		Hashtable seenAlready=null;
		if (s.distinct) {
			seenAlready = new Hashtable();
		}
		while (iter.hasNext()) {
			Row ro = (Row) iter.next();
			count++;
			if (!isValid(ro) || !where.includes(ro, this)) {
				// ro doesn't fit whereClause
				iter.remove();
				copyFlag = true;
			} else if (s.distinct) {
				if (seenAlready.containsKey(ro)) {
					// ro fits whereClause, but is a repeat row, and
					// select clause calls for distinct output
					iter.remove();
					copyFlag = true;
				} else {
					// ro is a keeper, but detect future repeats
					seenAlready.put(ro, this);
				}
			}
		}
		// reshape to columnSpec and recopy, if necessary
		if (!cs.equals(rs.getColumnSpec())) {
			rs = new ResultSetNarrow(cs, rs);
		} else if (copyFlag) {
			// by fiddling with the ResultSet, we may have corrupted its
			// precomputed metadata. Recreate the ResultSet with only the data.
			rs = new ResultSetImpl(rs);
		}
			// else rs is just fine as is

		// sort, if specified in Select statement
		rs = s.order(rs);

		// System.err.println("request processing: "+timer);
		// Tools.DumpProf.dump();
		timingexp.Timeline.timePoint("exit  evaluateSelect");
		return rs;
	}

	protected boolean isValid(Row ro) {
		// TODO: this is reaaally slow. It would be faster
		// to specialize the verification while loop in evaluateSelect().
		if (ro instanceof Relational) {
			return ((Relational) ro).valid;
		} else {
			FromClause fc = ro.getColumnSpec().getFromClause();
			int nt = fc.getNumTables();
			for (int i=0; i<nt; i++) {
				if (!ro.getTable(fc, i).valid)
					return false;
			}
			return true;
		}
	}

	/**
	 * Optimization for Where clauses.
	 * Given a Where clause, the FromClause it is scoped over, and some
	 * input superset, (quickly) compute a (possibly not-tight) superset
	 * of the possible matching rows fitting the FromClause.<p>
	 *
	 * The superset can be a loose bound in two ways: First, in the
	 * obvious way, it can explicitly list more rows that actually match
	 * the request. Second, it can have a weaker type (fromClause). So if
	 * fc=<em>tableA,tableB</em>, but the superset ResultSet's
	 * getFromClause()=<em>tableA</em>, then the superset contains the join of
	 * its rows with every row in <em>tableB</em>,
	 * which is a shorthand for a lot of rows.<p>
	 *
	 * The latter loose bound is used when computing joins, in fact. One
	 * whereClause finds a condition on one table, and expresses it as
	 * described above (compactly, listing only the matching rows of tableA).
	 * Then the whereJoin() clause can index tableB on the joined column
	 * of tableA, filling out the type (fromClause) of the bounding set,
	 * and therefore making the bound tighter (because it doesn't end
	 * up listing every possible row of tableB with every row of the input
	 * superset.)<p>
	 *
	 * Often these routines compute an actually-tight superset, at least for
	 * the returned fromClause. However, right now the semantics is that
	 * whatever results are returned, they are first expanded to the full
	 * requested fromClause (by joining in unmentioned tables), then every
	 * row of the result is forced through the where expression to verify
	 * that it matches.<p>
	 *
	 * @todo if we can ensure that our indexing is good, we may be able
	 * to eliminate that final sieve. We could either always return tight
	 * bounding sets, or by including a "needs-verify-pass" bit, so that
	 * we can have whereClauses with helpful but not tight bound*() functions,
	 * that only incur the verify pass slowdown when they are involved in
	 * a query.
	 */
	public ResultSet boundingSuperset(Where w, FromClause fc, ResultSet input) {
		// System.out.println("processing where\n"+w.indentedString(2));
		// Timer t = new Timer();

		ResultSet result;
		if (w instanceof WhereAnd) {
			result = boundAnd((WhereAnd) w, fc, input);
		} else if (w instanceof WhereLiteral) {
			result = boundLiteral((WhereLiteral) w, fc, input);
		} else if (w instanceof WhereJoin) {
			result = boundJoin((WhereJoin) w, fc, input);
		} else if (w instanceof WhereConstant) {
			result = boundConstant((WhereConstant) w, fc, input);
		} else if (w instanceof WhereOr) {
			result = boundOr((WhereOr) w, fc, input);
		} else if (w instanceof WhereNot) {
			result = boundNot((WhereNot) w, fc, input);
		} else if (w instanceof WhereIn) {
			result = boundIn((WhereIn) w, fc, input);
		} else if (w instanceof WhereAlways) {
			// hah, that one's easy.
			result = input;
		} else {
			throw new RuntimeException(
				"Unknown Where subclass "+w.getClass()+". Bummer.");
		}
//		System.err.println((result==null?0:result.size())
//			+"-item bounding set for "+w);
// 		System.out.println("--elasped time "+t.wallTime());
		return result;
	}

	public ResultSet boundAnd(WhereAnd w, FromClause fc, ResultSet input) {
		ResultSet r1 = boundingSuperset(w.w1, fc, input);
		return boundingSuperset(w.w2, fc, r1);
	}

	public ResultSet boundLiteral(WhereLiteral w,
		FromClause fc, ResultSet input) {

		// literal equality/likeness
		ResultSet r=null;
		// TODO: get some advantage from input when it just needs to
		// be joined.
		if (input!=null
			&& input.getColumnSpec().supports(w.getFieldDescriptor())) {
			// filter input set
			// TODO: It might well be faster, if there's an index, to
			// look it up and intersect it with 'input'.
			r = input;
			// System.out.println("filtering an input set of size "+r.size()+" for "+w);
		} else {
			Class t = w.getFieldDescriptor().getDeclaringClass();
			String fn = fc.getName(t);
			if (w instanceof WhereEquals) {
				r = primeFromIndexWE((WhereEquals) w, fn);
			} else if (w instanceof WhereLike) {
				r = primeFromIndexWL((WhereLike) w, fn);
			}
			if (r==null) {
				System.out.println("indexing "+w+" failed; filtering all.");
				// filter entire table
				Class cl = ((FieldDescriptor)w.getChild(0)).getDeclaringClass();
				r = getUniverse(cl, fc.getName(cl));
			}
			// if we trust index, we could stop here when r!=null
		}
		// TODO: is this filter necessary? This whole function is optimistic...
		// ...but it helps a lot if a table isn't indexed, and we're
		// joining. Better to brute-force 'n' entries than join 'n x m' and
		// brute-force that. There may be some general principal by which
		// we can detect when it'll pay off to periodically do partial
		// filters mid-evaluation. Perhaps just before any join?
		ResultSetImpl rso= new ResultSetImpl(r.getFromClause());
		Enumeration e = r.getEnumeration();
		while (e.hasMoreElements()) {
			Row ro = (Row) e.nextElement();
			if (w.includes(ro, this)) {
				rso.addMember(ro);
			}
		}
		return rso;
	}

	public ResultSet boundJoin(WhereJoin wj, FromClause fc, ResultSet input) {
		// input type matters
		FieldDescriptor f1 = wj.f1;
		FieldDescriptor f2 = wj.f2;
		// I guess it's an assumption that f1 and f2 come from different
		// tables, or this wouldn't be a join.
		Assert.assert(f1.getDeclaringClass() != f2.getDeclaringClass());
		if (input==null) {
			// do the full join (ack!)
			System.out.println("ack, doing full join.");
			Class c1 = f1.getDeclaringClass();
			ResultSet r1 = getUniverse(c1, fc.getName(c1));
			Class c2 = f2.getDeclaringClass();
			ResultSet r2 = getUniverse(c2, fc.getName(c2));
			return ResultSetImpl.cross(r1, r2);
		} else if (input.getColumnSpec().findField(f1)>=0
			&& input.getColumnSpec().findField(f2)>=0) {
			// input has both fields needed by wj
			// filter input according to the join (equality) condition
			ResultSetImpl ro = new ResultSetImpl(input.getFromClause());
			Enumeration e = input.getEnumeration();
			while (e.hasMoreElements()) {
				Object inputRow = e.nextElement();
				if (wj.includes((Relational) inputRow, this)) {
					ro.addMember(inputRow);
				}
			}
			return ro;
		} else {
			// input is one of the two types the equality references --
			// use the index on the rest, if we can.
			// arrange for f1 to be the field that appears in the input rows
			ColumnSpec ics = input.getColumnSpec();
			int f1i = ics.findField(f1);
			if (f1i<0) {
				FieldDescriptor tmp = f2;
				f2 = f1;
				f1 = tmp;
				f1i = input.getColumnSpec().findField(f1);
				if (f1i<0) {
					throw new RuntimeException(
						"Unexpected: input helps neither join field.");
				}
			}
			// for each f1, we'll want to find the f2 that equals it.
			Hashtable index = (Hashtable) indices.get(f2);
			if (index==null) {
				// pre-"index" input
				index = createIndex(f1, input);
			}
			Class t = f2.getDeclaringClass();
			int addingIndex = fc.getIndex(t);
			String addingName = fc.getName(addingIndex);
			FromClause fc2 = FromClause.create(addingName, t);
			FromClause ofc = FromClause.union(input.getFromClause(), fc2);
			ResultSetImpl ors = new ResultSetImpl(ofc);
			Iterator inputIter = input.iterator();
			while (inputIter.hasNext()) {
				Row row = (Row) inputIter.next();	// 1 row of input
				Object value = ics.getField(row, f1i);
				SmallHashset shs = (SmallHashset) index.get(value);
				if (shs!=null) {
					Iterator indexIter = shs.getKeyIterator();
					while (indexIter.hasNext()) {
						ors.addMember(new BasicRow(ofc, row, addingIndex,
							(Relational) indexIter.next()));
					}
				}
			}
			return ors;
		}
	}

	public ResultSet boundConstant(WhereConstant wc,
		FromClause fc, ResultSet input) {
		FromClause ifc = input.getFromClause();
		int index = ifc.getIndex(wc.tableName);
		Assert.assert(index<0,
			"hmm, result set already *has* our table in it?");
		// "join" input to our single-row virtual table
		FromClause ofc = FromClause.union(ifc, wc.pseudoFromClause);

		ResultSetImpl ors = new ResultSetImpl(ofc);
		Iterator inputIter = input.iterator();
		int outputIndex = ofc.getIndex(wc.tableName);
		while (inputIter.hasNext()) {
			Row row = (Row) inputIter.next();	// 1 row of input
			ors.addMember(new BasicRow(ofc, row, outputIndex, wc.constant));
		}
		return ors;
	}

	public ResultSet boundOr(WhereOr wor, FromClause fc, ResultSet input) {
		// do both kids naively and union them
		ResultSet r1 = boundingSuperset(wor.w1, fc, input);
		ResultSet r2 = boundingSuperset(wor.w2, fc, input);
		if (r1==null || r2==null) {
			// kids aren't enough help for me to be helpful.
			return null;
		}
		if (!r1.getFromClause().equals(r2.getFromClause())) {
			// both kids had a suggestion, but connecting them
			// would involve pulling off a join, so let's bail out
			// instead.
			return null;
			// TODO: If one fc is a subset of the other, we can
			// do a loose union, without a join, by returning the union
			// over the smaller fromClause, ignoring the extra information
			// in the longer rows.
		}
		// both kids provided a bounding superset over the same fromClause,
		// so their union is my bounding superset.
		ResultSetImpl ro = new ResultSetImpl(r1);
		Enumeration e = r2.getEnumeration();
		while (e.hasMoreElements()) {
			Row o = (Row) e.nextElement();
			if (!r1.hasMember(o)) {
				ro.addMember(o);
			}
		}
		return ro;
	}

	public ResultSet boundNot(WhereNot wnot, FromClause fc, ResultSet input) {
		// do kid naively and invert him
		ResultSet rk = boundingSuperset(wnot.w1, fc, null);
		if (rk==null) {
			return null;	// okay, then we'll be of no help, either.
		}
		ResultSet ru = getUniverse(rk.getFromClause());
		Enumeration e = ru.getEnumeration();
		ResultSetImpl ro = new ResultSetImpl(rk.getFromClause(), new Vector());
		while (e.hasMoreElements()) {
			Row o = (Row) e.nextElement();
			if (!rk.hasMember(o)) {
				ro.addMember(o);
			}
		}
		return ro;
	}

	public ResultSet boundIn(WhereIn win, FromClause fc, ResultSet input) {
		ResultSet r1 = win.getQuery().evaluate(this);
			// TODO: might be an optimization for subquery using input set
			// TODO: implement filtering the input set based on the
			// subquery. -- only if that helps, since we run a final
			// recheck of the query at the end.
		// recast r's fromClause "type" to be that of
		// win.fd.getDeclaringClass(), since its rows will be playing that
		// role in the final (outside) fromClause.
		ColumnSpec cs1 = r1.getColumnSpec();
		Class ic = cs1.getType(0);
		Class oc = win.getFieldDescriptor().getType();
		// System.err.println("ic = "+ic);
		// System.err.println("oc = "+oc);
		Tools.Assert.assert(ic==oc);
		// TODO: select Body.text from Body where Body.msg IN ...
		// won't work by this trivial casting -- we'll need to index
		// Body on each 'msg' in r1 to create r2.
		// FromClause ofc = FromClause.createAnonymous(oc);
		Class t = win.getFieldDescriptor().getDeclaringClass();
		FromClause ofc = FromClause.create(fc.getName(t), t);
		ResultSetImpl r2 = new ResultSetImpl(ofc);
		FieldDescriptor ofd = win.getFieldDescriptor();
		Hashtable index = (Hashtable) indices.get(ofd);
		if (index!=null) {
			Iterator iter = r1.iterator();
			while (iter.hasNext()) {
				Object o = cs1.getField((Row) iter.next(), 0);
				// find the member of ofc with primaryKey o.
				SmallHashset list = (SmallHashset) index.get(o);
				if (list!=null) {
					Iterator iter2 = list.getKeyIterator();
					if (iter2.hasNext()) {
						// members of index are of type ofc.
						r2.addMember(iter2.next());
					}
				}
			}
		} else {
			Class c = ofd.getDeclaringClass();
			ResultSet allSuitors = getUniverse(c, fc.getName(c));
			// no index -- gotta find members the hard way
			Iterator iter = r1.iterator();
			while (iter.hasNext()) {
				Object o = cs1.getField((Row) iter.next(), 0);
				Iterator suitorIter = allSuitors.iterator();
				while (suitorIter.hasNext()) {
					Row so = (Row) suitorIter.next();
					if (o.equals(ofd.get(so))) {
						r2.addMember(so);
					}
				}
			}
		}
		return r2;
	}

	// TODO -- could be implemented using fillByJoin(emptyResultSet, fc)?
	protected ResultSet getUniverse(FromClause fc) {
		if (fc.getNumTables()==1) {
			return getUniverse(fc.getTable(0), fc.getName(0));
		} else {
			// multiple classes in the set. Get one set, cross it with
			// the next... This is a Painful Join. We're doing well
			// if we can avoid this code path.
			FromClause rest = FromClause.trimOne(fc);
			ResultSet ra = getUniverse(fc.getTable(0), fc.getName(0));
			ResultSet rb = getUniverse(rest);
			return ResultSetImpl.cross(ra, rb);
		}
	}

	protected ResultSet fillByJoin(ResultSet rsi, FromClause fc) {
		// look for a missing table, fill it in, then recurse.
		FromClause rfc = rsi.getFromClause();
		Class fillTable = null;
		String fillName = null;
		for (int i=0; i<fc.getNumTables(); i++) {
			if (!rfc.hasTable(fc.getTable(i))) {
				// found our mark
				fillTable = fc.getTable(i);
				fillName = fc.getName(i);
				break;
			}
		}
		if (fillTable==null) {
			// all tables present. Done!
			return rsi;
		} else {
			// Join rsi with fillTable.
			ResultSet rb = getUniverse(fillTable, fillName);
			return ResultSetImpl.cross(rsi, rb);
		}
	}

	protected ResultSet getUniverse(Class c, String fromName) {
		HashMap objectList = (HashMap) tableRowsByClass.get(c);
		if (objectList==null) {
			objectList = new HashMap();
		}
		return new ResultSetImpl(FromClause.create(fromName, c),
			objectList.values());
	}

	////////////////////////////////////////////////////////////
	// indexing code & data
	////////////////////////////////////////////////////////////

	Hashtable indices = new Hashtable();
	// indices.get(fd) = index for a specific FieldDescriptor
	// indices.get(fd).get(target) = index (form of a hashtable, keys
	// significant, values ignored) of objects whose 'fd' equals 'target'.
	// Also, indices.get(field).get(LikeString.get(target)) = index of the
	// same form of objects whose 'field' contains 'target'.

	HashMap indexInstructions = new HashMap();
	// indexInstructions.get(class) = HashSet whose members are
	// FieldDescriptors of class that are indexed.

	static final int minLikeIndexLength = 4;
	static final int maxLikeIndexLength = 4;

	static Object likeIndex = new Object();
	// indices.get(field).get(likeIndex) is a like index (Hashtable).
	// indices.get(field).get(likeIndex).get(LikeHash.get((String) s))
	//	is a Hashtable whose keys are objects whose 'field' may contain a
	//	String containing s.

	//
	// create index data structures
	//

	/**
	 * iterates over all members of the class(es) that declare field,
	 * indexing what they point to.
	 */
	public void createIndex(FieldDescriptor fd) {
		checkAuth();

		if (indices.get(fd)!=null) {
			return;
		}
		indices.put(fd,
			createIndex(fd, getUniverse(fd.getDeclaringClass(), "Anonymous")));
			// name "Anonymous" will get discarded; we are only using
			// the ResultSet here for the rows in it, not its fromClause

		// Note instructions for future insert/updates.
		HashSet fdSet =
			(HashSet) indexInstructions.get(fd.getDeclaringClass());
		if (fdSet==null) {
			fdSet = new HashSet();
			indexInstructions.put(fd.getDeclaringClass(), fdSet);
		}
		fdSet.add(fd);
	}

	protected Hashtable createIndex(FieldDescriptor fd, ResultSet rs) {
		// A method where I can set my LikeHash parameters
		// (and at one point I tried creating the index several times
		// with varying parameters)
		LikeHash lh = null;
		if (String.class.isAssignableFrom(fd.getType())) {
			lh = new LikeHash(4, 13, 2);
			System.err.println("Like index of "+fd.getType()+" for "+fd);
		} else {
		}
		return createIndex(fd, rs, lh);
	}

	protected Hashtable createIndex(FieldDescriptor fd, ResultSet rs,
		LikeHash lh) {

		Memory mt=null;
		if (testingIndex) {
			mt = new Memory();
		}

		// The real work
		// TODO: For now, this only works with ResultSets whose
		// rows are Objects, not BasicRows.
		Hashtable index = new Hashtable();
		if (lh!=null) {
			index.put(likeIndex, lh);
				// drop a hint in the table that all Strings should get
				// chopped up for a Like substring index.
		}
		Enumeration e = rs.getEnumeration();

		System.out.println("Creating index of "+fd);
		ProgressBar pb = null;
		int count=0;
		if (rs.size()>0) {
			pb = new ProgressBar(rs.size());
		}

		while (e.hasMoreElements()) {
			Relational r = (Relational) e.nextElement();
			Object o = fd.get(r);
			indexOneField(index, r, o);
			if (pb!=null) {
				pb.update(++count);
			}
		}
		if (pb!=null) {
			pb.done();
		}

		if (testingIndex && (lh != null)) {
			// generate report
			int memused = (int) mt.getMemory();
			
			int skip = lh.getSkip();
			System.out.println("----index efficacy report---------------");
			System.out.println("likeIndexBits = "+lh.likeIndexBits);
			System.out.println("likeKeepBits = "+lh.likeKeepBits);
			System.out.println("skip = "+skip);
			NumberFormat tenths = NumberFormat.getInstance();
			tenths.setMaximumFractionDigits(1);
	
			int hcount = lh.theLists.length;
			int htotal = 0;
			if (lh!=null) {
				for (int li=0; li<hcount; li++) {
					SmallHashset kcol = lh.theLists[li];
					htotal += (kcol==null ? 0 : kcol.size());
				}
			}
			System.out.println("Index of "+fd+" consumes "+memused/1024+"k");
			System.out.println("avg filter size "+tenths.format(((double)htotal)*skip/hcount));
			System.out.println("metric: "+tenths.format(((double)memused)*htotal*skip/hcount/1024/1024));
			// warning: memory-based metrics are meaningless if WhereEquals hashing
			// is also turned on...
			System.out.println("----------------------------------------");
		}

		return index;
	}

	//
	// populate index data structures
	//

	public void indexOneField(Hashtable index, Relational r, Object target) {
		// index is the index for some Field f
		// Field f of Relational object r points to Object target
		indexOneValue(index, r, target);

		LikeHash lh;
		if ((target instanceof String)
			&& (lh = (LikeHash) index.get(likeIndex))!=null) {
			// need to index every substring of this field of
			// length <= maxLikeIndexLength
			int skip = lh.getSkip();
			String s = (String) target;
			int slen = s.length();
			for (int l=minLikeIndexLength; l<=maxLikeIndexLength; l++) {
				for (int o=0; o<=slen-l; o+=skip) {
					lh.put(s.substring(o, o+l), r);
				}
			}
		}
	}

	public void indexOneValue(Hashtable index, Relational r, Object target) {
		SmallHashset list = (SmallHashset) index.get(target);
		if (list==null) {
			list = new SmallHashset(7);
			index.put(target, list);
		}
		list.put(r);
	}

	//
	// employ index data structures
	//

	public ResultSet primeFromIndexWE(WhereEquals we, String fromName) {
		if (we.fd instanceof FieldDescriptorPrimary) {
			// weirdo.
			// TODO: is the right thing to return the object we.o?
			return null;
		}
		Hashtable index = (Hashtable) indices.get(we.fd);
		if (index != null) {
			SmallHashset list = (SmallHashset) index.get(we.o);
			Vector objectList = new Vector();
			if (list!=null) {
				Iterator iter = list.getKeyIterator();
				while (iter.hasNext()) {
					objectList.addElement(iter.next());
				}
			}
			// result is a primed objectList, so we don't have to
			// check *every* object. But indices may contain stale
			// references, so we still have to iterate over data
			// found in index.
			return new ResultSetImpl(
				FromClause.create(fromName, we.fd.getDeclaringClass()),
				objectList);
		}
		return null;
	}

	public ResultSet primeFromIndexWL(WhereLike wl, String fromName) {
		if (!(wl.fd instanceof FieldDescriptorField)) {
			// looking for a String in a primary/foreign key -- but
			// keys are Relational objects, which Strings are not,
			// so the answer is an empty set.
			System.err.println("Warning: WhereLike on key: "+wl);
			return new ResultSetImpl(
				FromClause.create(fromName, wl.fd.getDeclaringClass()));
		}
		Hashtable baseIndexH = (Hashtable) indices.get(wl.fd);
		LikeHash likeHash;
		if ((baseIndexH != null)
			&& ((likeHash = (LikeHash) baseIndexH.get(likeIndex))!=null)) {
			int skip = likeHash.getSkip();
			if (wl.s.length()<minLikeIndexLength+skip-1) {
				// field too short to index -- index points to most rows!
				return null;
			}
			// if skip > 1, need to hash on all possible alignments of
			// the substring.
			// TODO: if the query string is long, we could try multiple
			// alignments and use only the ones with the shortest index
			// results
			HashSet objectList = new HashSet();
			for (int align=0; align<skip; align++) {
				SmallHashset list = likeHash.get(wl.s.substring(align));
				if (list == null) {
					continue;	// no objects found at this alignment
				}
				Iterator iter = list.getKeyIterator();
				while (iter.hasNext()) {
					objectList.add(iter.next());
				}
			}
			// result is a primed objectList, so we don't have to
			// check *every* object. But indices may contain stale
			// references, or substring matches that don't actually
			// indicate a whole match, so we still have to iterate
			// over data found in index to discard false positives.
			return new ResultSetImpl(
				FromClause.create(fromName, wl.fd.getDeclaringClass()),
				objectList);
		}
		return null;
	}

	public void noop()
		throws RemoteException {
		checkAuth();
		System.err.println(getClass()+" noop");
	}

	/**
	 * The principal that controls this service.
	 */
	SDSIPrincipal serverIssuer=null;

	/**
	 * The tag required to be able to manipulate this service.
	 */
	Tag requestTag;

	void initAuthorization(SDSIPrincipal serverIssuer) {
		try {
			SexpList databaseTag
				= (new SexpList()).append("database").append("mine");
			SexpList sl = (new SexpList()).append("tag").append(databaseTag);
			this.requestTag = new Tag(sl);
			this.serverIssuer = serverIssuer;
		} catch (SexpParseException ex) {
			ex.printStackTrace();
		}
	}

	/**
	 * @throws the unchecked SfNeedAuthorizationException when
	 * sufficient proof of authorization has not been presented.
	 */
	void checkAuth() {
		if (serverIssuer==null) {
			// no-authorization mode
			return;
		}

		ssh.RSA.RSAKey k = SSHSocket.whoCalledMe();
		SDSIRSAPublicKey subject = new SDSIRSAPublicKey(k);

		try {
			Proof proof = OneLineCacheRecipient.getCachedProof(subject);
			if (proof==null) {
				throw new InvalidProofException("no proof found");
			}
			proof.verify(serverIssuer, subject, requestTag);
			return;
		} catch (InvalidProofException ex) {
			System.err.println("internalDatabase: Need auth that "
				+Prover2.staticGetName(subject)
				+" => "
				+Prover2.staticGetName(serverIssuer));
			throw new SfNeedAuthorizationException(serverIssuer,
				subject, requestTag, OneLineCacheRecipient.getRecipient(),
				ex.toString());
		}
	}
}
