Index: /query-answering/src/main/java/gr/ntua/ece/cslab/modissense/dataoriented/QueryAnswering.java
===================================================================
--- /query-answering/src/main/java/gr/ntua/ece/cslab/modissense/dataoriented/QueryAnswering.java (revision 750)
+++ /query-answering/src/main/java/gr/ntua/ece/cslab/modissense/dataoriented/QueryAnswering.java (revision 751)
@@ -20,5 +20,4 @@
 
 import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -34,4 +33,6 @@
 	public static Integer THREADS = 10;
 	public static boolean AUTO_THREADS=false; 
+	public static Integer CHOP_LOWER_RANKS=-1;
+	public static boolean DEBUG=false;
 
 	
@@ -119,13 +120,14 @@
 	
 	public void executeQuery() throws IOException, InterruptedException{
+		long start = System.currentTimeMillis();
 		List<POIList> list = this.getDataFromDB();
-		
+		if(DEBUG)System.out.println("Fetching POI's took:\t"+(System.currentTimeMillis()-start)+" ms");
+		
+		start = System.currentTimeMillis();
 		Map<Long, List<POI>> map = new HashMap<Long, List<POI>>();
 		for(POIList pl:list){
 			for(POI p:pl.getPOIs()){
-				if(this.POIisValid(p)){
 					if(map.get(p.getId())==null){
 						map.put(p.getId(), new LinkedList<POI>());
-					}
 					map.get(p.getId()).add(p);
 				}
@@ -133,4 +135,6 @@
 			}
 		}
+		if(DEBUG)System.out.println("Grouping POIs took:\t"+(System.currentTimeMillis()-start)+" ms");
+		start = System.currentTimeMillis();
 		
 		TreeMap<Double, POI> sorted = new TreeMap<Double, POI>();
@@ -152,4 +156,5 @@
 			sorted.put(rank, newPOI);
 		}
+		if(DEBUG)System.out.println("Sorting POIs took:\t"+(System.currentTimeMillis()-start)+" ms");
 		
 		int count = 0;
@@ -181,9 +186,16 @@
 				sublist = this.users.subList(beginIndex, endIndex);
 			}
-			
-			DataFetcherThread thread = new DataFetcherThread();
-			thread.setUsers(sublist);
-			thread.setTable(table);
-			threads.add(thread);
+			if(!sublist.isEmpty()){
+				DataFetcherThread thread = new DataFetcherThread();
+				thread.setUsers(sublist);
+				thread.setTable(this.table);
+				thread.setKeywords(this.keywords);
+				thread.setxFrom(xFrom);
+				thread.setxTo(xTo);
+				thread.setyFrom(yFrom);
+				thread.setyTo(yTo);
+				
+				threads.add(thread);
+			}
 		}
 		
@@ -208,23 +220,21 @@
 		leftCut = (partition==0?0l:this.cuts[partition-1]);
 		rightCut = (partition==this.cuts.length?Long.MAX_VALUE:this.cuts[partition]);
-		int smallIndex=0, largeIndex=(rightCut==Long.MAX_VALUE?this.users.size()-1:0);
-		while(this.users.get(++smallIndex)<leftCut);
+		
+		int smallIndex=0, largeIndex=0;
+		while(smallIndex<this.users.size() && this.users.get(smallIndex)<leftCut){
+			smallIndex+=1;
+		}
+		
+		if(leftCut==0)
+			smallIndex=0;
 		largeIndex = smallIndex;
-		while(this.users.get(++largeIndex)<rightCut && rightCut!=Long.MAX_VALUE);
+		while(largeIndex<this.users.size() && this.users.get(largeIndex)<rightCut && rightCut!=Long.MAX_VALUE){
+			largeIndex+=1;
+		}
+		if(rightCut == Long.MAX_VALUE)
+			largeIndex=this.users.size();
 		return this.users.subList(smallIndex, largeIndex);
 	}
-	
-	public boolean POIisValid(POI poi){
-		if(poi.getX()<this.xFrom || poi.getX() > this.xTo)
-			return false;
-		if(poi.getY()<this.yFrom || poi.getY() > this.yTo)
-			return false;
-		for(String s:keywords){
-			if(poi.getKeywords().contains(s))
-				return true;
-		}
-		return false;
-	}
-	
+		
 	public void close() throws IOException{
 		this.table.close();
@@ -274,9 +284,9 @@
 		if(getIndex(args, "--coordinates")!=-1){
 			validArgumentsCount+=1;
-			String[] values = args[getIndex(args, "--coordinates")+1].split(" ");
-			xFrom = new Double(values[0]);
-			yFrom = new Double(values[1]);
-			xTo = new Double(values[2]);
-			yTo = new Double(values[3]);
+//			String[] values = args[getIndex(args, "--coordinates")+1].split(" ");
+			xFrom = new Double(args[getIndex(args, "--coordinates")+1]);
+			yFrom = new Double(args[getIndex(args, "--coordinates")+2]);
+			xTo = new Double(args[getIndex(args, "--coordinates")+3]);
+			yTo = new Double(args[getIndex(args, "--coordinates")+4]);
 		}
 		
@@ -303,4 +313,10 @@
 			OUTPUT_RESULTS = new Integer(args[getIndex(args, "--output")+1]);
 		}
+		if(getIndex(args, "--chop")!=-1) {
+			CHOP_LOWER_RANKS = new Integer(args[getIndex(args, "--chop")+1]);
+		}
+		if(getIndex(args, "--debug")!=-1){
+			DEBUG = true;
+		}
 		
 		if(validArgumentsCount!=3) {
@@ -321,5 +337,11 @@
 					+ "--threads\t\tthreads initiated for hitting the db server"
 					+ "\n\t"
-					+ "--output\t\tdetermine the number of POIs chosen");
+					+ "--output\t\tdetermine the number of POIs chosen"
+					+ "\n\t"
+					+ "--chop\t\t\tdetermine how many places will remain after parsing from Db"
+					+ "\n\t"
+					+ "--show-results\t\tshow results on the screen"
+					+ "\n\t"
+					+ "--debug\t\t\tprint diagnostic message");
 			System.exit(1);
 		}
@@ -335,13 +357,13 @@
 		
 		long start = System.currentTimeMillis();
-//		System.out.print("Fetching data from database...");
-//		qa.getDataFromDB();
-//		System.out.format(" DONE! (%d ms)\n", (System.currentTimeMillis()-start));
-		for(int i=0;i<qa.cuts.length;i++)
-			qa.getSublistFromCuts(i);
 		qa.executeQuery();
-		System.out.println(qa.users.size()+"\t"+(System.currentTimeMillis()-start));
+		System.out.print((System.currentTimeMillis()-start));
 		qa.close();
-		System.err.println(qa.getResults());
+		if(getIndex(args, "--show-results")!=-1){
+			System.out.println();
+			for(POI p:qa.getResults()){
+			System.out.format("%d\t%.5f\t(%.2f,%.2f)\t%s\n", p.getId(), p.getScore(), p.getX(), p.getY(), p.getKeywords().toString());
+			}
+		}
 	}
 
Index: /query-answering/src/main/java/gr/ntua/ece/cslab/modissense/dataoriented/datagen/QuickDataGenerator.java
===================================================================
--- /query-answering/src/main/java/gr/ntua/ece/cslab/modissense/dataoriented/datagen/QuickDataGenerator.java (revision 749)
+++ /query-answering/src/main/java/gr/ntua/ece/cslab/modissense/dataoriented/datagen/QuickDataGenerator.java (revision 751)
@@ -106,7 +106,13 @@
 			admin.deleteTable(TABLE_NAME);
 		}
+		byte[][] splits = new byte[3][8];
+		for(int i=1;i<4;i++){
+			System.out.println("Split @ "+(i*(this.usersSize/4l)));
+			splits[i-1] =Bytes.toBytes(i*(this.usersSize/4l));
+		}
 		HTableDescriptor descripton = new HTableDescriptor(TABLE_NAME);
 		descripton.addFamily(new HColumnDescriptor(COLUMN_FAMILY.getBytes()));
-		admin.createTable(descripton);
+		
+		admin.createTable(descripton, splits);
 		admin.close();
 	}
Index: /query-answering/src/main/java/gr/ntua/ece/cslab/modissense/dataoriented/containers/POIList.java
===================================================================
--- /query-answering/src/main/java/gr/ntua/ece/cslab/modissense/dataoriented/containers/POIList.java (revision 748)
+++ /query-answering/src/main/java/gr/ntua/ece/cslab/modissense/dataoriented/containers/POIList.java (revision 751)
@@ -4,11 +4,17 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
 import java.util.zip.DataFormatException;
 import java.util.zip.Deflater;
 import java.util.zip.Inflater;
+
+import com.google.common.collect.MinMaxPriorityQueue;
 
 
@@ -35,4 +41,55 @@
 	}
 
+	public void chopLowScores(int highRanksToKeep) {
+		if(highRanksToKeep>=this.poiList.size())
+			return;
+		
+		List<Double> list = new LinkedList<Double>();
+		for(POI p:this.poiList)
+			list.add(p.getScore());
+		Collections.sort(list);
+		double pivot = list.get(highRanksToKeep-1);		
+		
+//		double pivot = this.getKthBiggerNumber(highRanksToKeep);
+		List<POI> newPoiList = new LinkedList<POI>();
+		for(POI p :this.poiList)
+			if(p.getScore()>=pivot)
+				newPoiList.add(p);
+		this.poiList = newPoiList;
+	}
+	
+//	private double getKthBiggerNumber(int k){
+//		MinMaxPriorityQueue<Double> queue = MinMaxPriorityQueue.create();
+//		for(POI p:this.poiList)
+//			queue.add(p.getScore());
+//		double pivot = 0;
+//		for(int i=0;i<k;i++)
+//			pivot = queue.pollLast();
+//		return pivot;
+//	}
+	public void clearList(double xFrom, double yFrom, double xTo, double yTo){
+		List<POI> list = new LinkedList<POI>();
+		for(POI p:this.poiList){
+			double x = p.getX();
+			double y = p.getY();
+			if(x>=xFrom && x<=xTo && y>=yFrom && y<=yTo)
+				list.add(p);
+		}
+		this.poiList = list;
+	}
+	
+	public void clearList(List<String> keywords) {
+		List<POI> list = new LinkedList<POI>();
+		for(POI p:this.poiList){
+			for(String s:keywords){
+				if(p.getKeywords().contains(s)){
+					list.add(p);
+					break;
+				}
+			}
+		}
+		this.poiList = list;
+	}
+	
 	@Override
 	public void parseBytes(byte[] bytes) {
@@ -48,5 +105,4 @@
 			this.poiList.add(poi);
 		}
-
 	}
 
@@ -164,5 +220,3 @@
 	}
 
-	
-
 }
Index: /query-answering/src/main/java/gr/ntua/ece/cslab/modissense/dataoriented/threads/DataFetcherThread.java
===================================================================
--- /query-answering/src/main/java/gr/ntua/ece/cslab/modissense/dataoriented/threads/DataFetcherThread.java (revision 749)
+++ /query-answering/src/main/java/gr/ntua/ece/cslab/modissense/dataoriented/threads/DataFetcherThread.java (revision 751)
@@ -5,10 +5,14 @@
 
 import java.io.IOException;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Set;
 
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.util.Bytes;
 
@@ -18,7 +22,11 @@
 	private List<Long> users;
 	private List<POIList> results;
-
+	private double density;
+
+	private double xFrom, xTo, yFrom, yTo;
+	private List<String> keywords;
 
 	private long totalTime=0, fetchTime=0, transformTime=0;
+	private Set<Long> usersSet;
 	public DataFetcherThread() {
 		super();
@@ -40,4 +48,5 @@
 	public void setUsers(List<Long> users) {
 		this.users = users;
+		this.usersSet = new HashSet<Long>(this.users);
 	}
 
@@ -56,4 +65,44 @@
 	public long getTransformTime() {
 		return transformTime;
+	}
+
+	public double getxFrom() {
+		return xFrom;
+	}
+
+	public void setxFrom(double xFrom) {
+		this.xFrom = xFrom;
+	}
+
+	public double getxTo() {
+		return xTo;
+	}
+
+	public void setxTo(double xTo) {
+		this.xTo = xTo;
+	}
+
+	public double getyFrom() {
+		return yFrom;
+	}
+
+	public void setyFrom(double yFrom) {
+		this.yFrom = yFrom;
+	}
+
+	public double getyTo() {
+		return yTo;
+	}
+
+	public void setyTo(double yTo) {
+		this.yTo = yTo;
+	}
+
+	public List<String> getKeywords() {
+		return keywords;
+	}
+
+	public void setKeywords(List<String> keywords) {
+		this.keywords = keywords;
 	}
 
@@ -62,13 +111,45 @@
 		super.start();
 		this.totalTime = System.currentTimeMillis();
+		double range = this.users.get(this.users.size()-1)-this.users.get(0)*1.0;
+		this.density = (range!=0?this.users.size()/(range*1.0):-1);
+		//System.out.println("Density:\t"+this.density);
 	}
 	@Override
 	public void run() {
 		try {
-			List<Get> gets = new LinkedList<Get>();
-			for(Long userId:this.users){
-				Get get = new Get(Bytes.toBytes(userId));
-				gets.add(get);
-				if(gets.size()%QueryAnswering.GET_BUFFER==0) {
+			if(this.density<0.50 && !Double.isInfinite(this.density)){
+				List<Get> gets = new LinkedList<Get>();
+				for(Long userId:this.users){
+					Get get = new Get(Bytes.toBytes(userId));
+					gets.add(get);
+					if(gets.size()%QueryAnswering.GET_BUFFER==0) {
+						long fetchTime = System.currentTimeMillis();
+						long transformTime = 0;
+						Result[] results;
+						results = table.get(gets);
+						for(Result r: results){
+							byte[] bytes = r.getValue(QueryAnswering.COLUMN_FAMILY.getBytes(), QueryAnswering.QUALIFIER.getBytes());
+							if(bytes!=null){
+								long start = System.currentTimeMillis();
+								POIList poilist = new POIList();
+								poilist.parseCompressedBytes(bytes);
+								
+								poilist.clearList(keywords);				//apply the query constraints..
+								poilist.clearList(xFrom, yFrom, xTo, yTo);	//apply the query constraints..
+								if(QueryAnswering.CHOP_LOWER_RANKS>-1)		//chop if possible
+									poilist.chopLowScores(QueryAnswering.CHOP_LOWER_RANKS);
+								
+								this.results.add(poilist);
+								transformTime=System.currentTimeMillis()-start;
+							} else {
+								System.out.println("Null byte array");
+							}
+						}
+						this.fetchTime += (System.currentTimeMillis()-fetchTime) - transformTime;
+						this.transformTime+=transformTime;
+						gets.clear();
+					}
+				}
+				if(gets.size()!=0) {
 					long fetchTime = System.currentTimeMillis();
 					long transformTime = 0;
@@ -81,34 +162,42 @@
 							POIList poilist = new POIList();
 							poilist.parseCompressedBytes(bytes);
+
+							poilist.clearList(keywords);				//apply the query constraints..
+							poilist.clearList(xFrom, yFrom, xTo, yTo);	//apply the query constraints..
+							if(QueryAnswering.CHOP_LOWER_RANKS>-1)		//chop if possible
+								poilist.chopLowScores(QueryAnswering.CHOP_LOWER_RANKS);
+							
 							this.results.add(poilist);
 							transformTime=System.currentTimeMillis()-start;
-						} else {
-							System.out.println("Null byte array");
 						}
 					}
 					this.fetchTime += (System.currentTimeMillis()-fetchTime) - transformTime;
 					this.transformTime+=transformTime;
-					gets.clear();
 				}
+			} else {
+				Scan scanner = new Scan();
+				scanner.setStartRow(Bytes.toBytes(this.users.get(0)));
+				scanner.setStopRow(Bytes.toBytes(this.users.get(this.users.size()-1)));
+				scanner.setCaching(1000);
+				ResultScanner rs = this.table.getScanner(scanner);
+				Result r = rs.next();
+				while(r!=null){
+					Long id = Bytes.toLong(r.getRow());
+					if(this.usersSet.contains(id)){
+						byte[] value = r.getValue(QueryAnswering.COLUMN_FAMILY.getBytes(), QueryAnswering.QUALIFIER.getBytes());
+						POIList poilist = new POIList();
+						poilist.parseCompressedBytes(value);
+						
+						poilist.clearList(keywords);				//apply the query constraints..
+						poilist.clearList(xFrom, yFrom, xTo, yTo);	//apply the query constraints..
+						if(QueryAnswering.CHOP_LOWER_RANKS>-1)		//chop if possible
+							poilist.chopLowScores(QueryAnswering.CHOP_LOWER_RANKS);
+						
+						this.results.add(poilist);
+					}
+					r = rs.next();
+				}
+				rs.close();
 			}
-			if(gets.size()!=0) {
-				long fetchTime = System.currentTimeMillis();
-				long transformTime = 0;
-				Result[] results;
-				results = table.get(gets);
-				for(Result r: results){
-					byte[] bytes = r.getValue(QueryAnswering.COLUMN_FAMILY.getBytes(), QueryAnswering.QUALIFIER.getBytes());
-					if(bytes!=null){
-						long start = System.currentTimeMillis();
-						POIList poilist = new POIList();
-						poilist.parseCompressedBytes(bytes);
-						this.results.add(poilist);
-						transformTime=System.currentTimeMillis()-start;
-					}
-				}
-				this.fetchTime += (System.currentTimeMillis()-fetchTime) - transformTime;
-				this.transformTime+=transformTime;
-			}
-
 		} catch (IOException e) {
 			e.printStackTrace();
