Stuart McCulloch | 26e7a5a | 2011-10-17 10:31:43 +0000 | [diff] [blame^] | 1 | package aQute.lib.index; |
| 2 | |
| 3 | import java.io.*; |
| 4 | import java.nio.*; |
| 5 | import java.nio.channels.*; |
| 6 | import java.nio.channels.FileChannel.MapMode; |
| 7 | import java.util.*; |
| 8 | |
| 9 | /** |
| 10 | * <pre> |
| 11 | * 0 -> 0, 122 -> 1 |
| 12 | * 123 -> 123, 244 -> 2 |
| 13 | * 245 -> 245, ... |
| 14 | * </pre> |
| 15 | * |
| 16 | * |
| 17 | */ |
| 18 | public class Index implements Iterable<byte[]> { |
| 19 | final static int LEAF = 0; |
| 20 | final static int INDEX = 1; |
| 21 | |
| 22 | final static int SIGNATURE = 0; |
| 23 | final static int MAGIC = 0x494C4458; |
| 24 | final static int KEYSIZE = 4; |
| 25 | |
| 26 | private FileChannel file; |
| 27 | final int pageSize = 4096; |
| 28 | final int keySize; |
| 29 | final int valueSize = 8; |
| 30 | final int capacity; |
| 31 | public Page root; |
| 32 | final LinkedHashMap<Integer, Page> cache = new LinkedHashMap<Integer, Index.Page>(); |
| 33 | final MappedByteBuffer settings; |
| 34 | |
| 35 | private int nextPage; |
| 36 | |
| 37 | class Page { |
| 38 | final int TYPE_OFFSET = 0; |
| 39 | final int COUNT_OFFSET = 2; |
| 40 | final static int START_OFFSET = 4; |
| 41 | final int number; |
| 42 | boolean leaf; |
| 43 | final MappedByteBuffer buffer; |
| 44 | int n = 0; |
| 45 | boolean dirty; |
| 46 | |
| 47 | Page(int number) throws IOException { |
| 48 | this.number = number; |
| 49 | buffer = file.map(MapMode.READ_WRITE, number * pageSize, pageSize); |
| 50 | n = buffer.getShort(COUNT_OFFSET); |
| 51 | int type = buffer.getShort(TYPE_OFFSET); |
| 52 | leaf = type != 0; |
| 53 | } |
| 54 | |
| 55 | Page(int number, boolean leaf) throws IOException { |
| 56 | this.number = number; |
| 57 | this.leaf = leaf; |
| 58 | this.n = 0; |
| 59 | buffer = file.map(MapMode.READ_WRITE, number * pageSize, pageSize); |
| 60 | } |
| 61 | |
| 62 | Iterator<byte[]> iterator() { |
| 63 | return new Iterator<byte[]>() { |
| 64 | Iterator<byte[]> i; |
| 65 | int rover = 0; |
| 66 | |
| 67 | public byte[] next() { |
| 68 | if (leaf) { |
| 69 | return k(rover++); |
| 70 | } |
| 71 | |
| 72 | return i.next(); |
| 73 | } |
| 74 | |
| 75 | public boolean hasNext() { |
| 76 | try { |
| 77 | if (leaf) |
| 78 | return rover < n; |
| 79 | else { |
| 80 | while (i == null || i.hasNext() == false) { |
| 81 | int c = (int) c(rover++); |
| 82 | i = getPage(c).iterator(); |
| 83 | } |
| 84 | return i.hasNext(); |
| 85 | } |
| 86 | } catch (IOException e) { |
| 87 | throw new RuntimeException(e); |
| 88 | } |
| 89 | |
| 90 | } |
| 91 | |
| 92 | public void remove() { |
| 93 | throw new UnsupportedOperationException(); |
| 94 | } |
| 95 | |
| 96 | }; |
| 97 | } |
| 98 | |
| 99 | void write() throws IOException { |
| 100 | buffer.putShort(COUNT_OFFSET, (short) n); |
| 101 | buffer.put(TYPE_OFFSET, (byte) (leaf ? 1 : 0)); |
| 102 | buffer.force(); |
| 103 | } |
| 104 | |
| 105 | int compare(byte[] key, int i) { |
| 106 | int index = pos(i); |
| 107 | for (int j = 0; j < keySize; j++, index++) { |
| 108 | int a = 0; |
| 109 | if (j < key.length) |
| 110 | a = key[j] & 0xFF; |
| 111 | |
| 112 | int b = buffer.get(index) & 0xFF; |
| 113 | if (a == b) |
| 114 | continue; |
| 115 | |
| 116 | return a > b ? 1 : -1; |
| 117 | } |
| 118 | return 0; |
| 119 | } |
| 120 | |
| 121 | int pos(int i) { |
| 122 | return START_OFFSET + size(i); |
| 123 | } |
| 124 | |
| 125 | int size(int n) { |
| 126 | return n * (keySize + valueSize); |
| 127 | } |
| 128 | |
| 129 | void copyFrom(Page page, int start, int length) { |
| 130 | copy(page.buffer, pos(start), buffer, pos(0), size(length)); |
| 131 | } |
| 132 | |
| 133 | void copy(ByteBuffer src, int srcPos, ByteBuffer dst, int dstPos, int length) { |
| 134 | if (srcPos < dstPos) { |
| 135 | while (length-- > 0) |
| 136 | dst.put(dstPos + length, src.get(srcPos + length)); |
| 137 | |
| 138 | } else { |
| 139 | while (length-- > 0) |
| 140 | dst.put(dstPos++, src.get(srcPos++)); |
| 141 | } |
| 142 | } |
| 143 | |
| 144 | long search(byte[] k) throws Exception { |
| 145 | int cmp = 0; |
| 146 | int i = n - 1; |
| 147 | while (i >= 0 && (cmp = compare(k, i)) < 0) |
| 148 | i--; |
| 149 | |
| 150 | if (leaf) { |
| 151 | if (cmp != 0) |
| 152 | return -1; |
| 153 | else |
| 154 | return c(i); |
| 155 | } else { |
| 156 | long value = c(i); |
| 157 | Page child = getPage((int) value); |
| 158 | return child.search(k); |
| 159 | } |
| 160 | } |
| 161 | |
| 162 | void insert(byte[] k, long v) throws IOException { |
| 163 | if (n == capacity) { |
| 164 | int t = capacity / 2; |
| 165 | Page left = allocate(leaf); |
| 166 | Page right = allocate(leaf); |
| 167 | left.copyFrom(this, 0, t); |
| 168 | left.n = t; |
| 169 | right.copyFrom(this, t, capacity - t); |
| 170 | right.n = capacity - t; |
| 171 | leaf = false; |
| 172 | set(0, left.k(0), left.number); |
| 173 | set(1, right.k(0), right.number); |
| 174 | n = 2; |
| 175 | left.write(); |
| 176 | right.write(); |
| 177 | } |
| 178 | insertNonFull(k, v); |
| 179 | } |
| 180 | |
| 181 | byte[] k(int i) { |
| 182 | buffer.position(pos(i)); |
| 183 | byte[] key = new byte[keySize]; |
| 184 | buffer.get(key); |
| 185 | return key; |
| 186 | } |
| 187 | |
| 188 | long c(int i) { |
| 189 | if (i < 0) { |
| 190 | System.out.println("Arghhh"); |
| 191 | } |
| 192 | int index = pos(i) + keySize; |
| 193 | return buffer.getLong(index); |
| 194 | } |
| 195 | |
| 196 | void set(int i, byte[] k, long v) { |
| 197 | int index = pos(i); |
| 198 | for (int j = 0; j < keySize; j++) { |
| 199 | byte a = 0; |
| 200 | if (j < k.length) |
| 201 | a = k[j]; |
| 202 | buffer.put(index + j, a); |
| 203 | } |
| 204 | buffer.putLong(index + keySize, v); |
| 205 | } |
| 206 | |
| 207 | void insertNonFull(byte[] k, long v) throws IOException { |
| 208 | int cmp = 0; |
| 209 | int i = n - 1; |
| 210 | while (i >= 0 && (cmp = compare(k, i)) < 0) |
| 211 | i--; |
| 212 | |
| 213 | if (leaf) { |
| 214 | if (cmp != 0) { |
| 215 | i++; |
| 216 | if (i != n) |
| 217 | copy(buffer, pos(i), buffer, pos(i + 1), size(n - i)); |
| 218 | } |
| 219 | set(i, k, v); |
| 220 | n++; |
| 221 | dirty = true; |
| 222 | } else { |
| 223 | long value = c(i); |
| 224 | Page child = getPage((int) value); |
| 225 | |
| 226 | if (child.n == capacity) { |
| 227 | Page left = child; |
| 228 | int t = capacity / 2; |
| 229 | Page right = allocate(child.leaf); |
| 230 | right.copyFrom(left, t, capacity - t); |
| 231 | right.n = capacity - t; |
| 232 | left.n = t; |
| 233 | i++; // place to insert |
| 234 | if (i < n) // ok if at end |
| 235 | copy(buffer, pos(i), buffer, pos(i + 1), size(n - i)); |
| 236 | set(i, right.k(0), right.number); |
| 237 | n++; |
| 238 | assert i < n; |
| 239 | child = right.compare(k, 0) >= 0 ? right : left; |
| 240 | left.dirty = true; |
| 241 | right.dirty = true; |
| 242 | this.dirty = true; |
| 243 | } |
| 244 | child.insertNonFull(k, v); |
| 245 | } |
| 246 | write(); |
| 247 | } |
| 248 | |
| 249 | public String toString() { |
| 250 | StringBuilder sb = new StringBuilder(); |
| 251 | try { |
| 252 | toString( sb, ""); |
| 253 | } catch (IOException e) { |
| 254 | e.printStackTrace(); |
| 255 | } |
| 256 | return sb.toString(); |
| 257 | } |
| 258 | |
| 259 | public void toString( StringBuilder sb, String indent ) throws IOException { |
| 260 | for (int i = 0; i < n; i++) { |
| 261 | sb.append(String.format("%s %02d:%02d %20s %s %d\n", indent, number, i, hex(k(i), 0, 4), leaf ? "==" : "->", c(i))); |
| 262 | if (! leaf ) { |
| 263 | long c = c(i); |
| 264 | Page sub = getPage((int)c); |
| 265 | sub.toString(sb,indent+" "); |
| 266 | } |
| 267 | } |
| 268 | } |
| 269 | |
| 270 | private String hex(byte[] k, int i, int j) { |
| 271 | StringBuilder sb = new StringBuilder(); |
| 272 | |
| 273 | while ( i < j) { |
| 274 | int b = 0xFF & k[i]; |
| 275 | sb.append(nibble(b>>4)); |
| 276 | sb.append(nibble(b)); |
| 277 | i++; |
| 278 | } |
| 279 | return sb.toString(); |
| 280 | } |
| 281 | |
| 282 | private char nibble(int i) { |
| 283 | i = i & 0xF; |
| 284 | return (char) ( i >= 10 ? i + 'A' - 10 : i + '0'); |
| 285 | } |
| 286 | |
| 287 | } |
| 288 | |
| 289 | public Index(File file, int keySize) throws IOException { |
| 290 | capacity = (pageSize - Page.START_OFFSET) / (keySize + valueSize); |
| 291 | RandomAccessFile raf = new RandomAccessFile(file, "rw"); |
| 292 | this.file = raf.getChannel(); |
| 293 | settings = this.file.map(MapMode.READ_WRITE, 0, pageSize); |
| 294 | if (this.file.size() == pageSize) { |
| 295 | this.keySize = keySize; |
| 296 | settings.putInt(SIGNATURE, MAGIC); |
| 297 | settings.putInt(KEYSIZE, keySize); |
| 298 | nextPage = 1; |
| 299 | root = allocate(true); |
| 300 | root.n = 1; |
| 301 | root.set(0, new byte[KEYSIZE], 0); |
| 302 | root.write(); |
| 303 | } else { |
| 304 | if (settings.getInt(SIGNATURE) != MAGIC) |
| 305 | throw new IllegalStateException("No Index file, magic is not " + MAGIC); |
| 306 | |
| 307 | this.keySize = settings.getInt(KEYSIZE); |
| 308 | if (keySize != 0 && this.keySize != keySize) |
| 309 | throw new IllegalStateException("Invalid key size for Index file. The file is " |
| 310 | + this.keySize + " and was expected to be " + this.keySize); |
| 311 | |
| 312 | root = getPage(1); |
| 313 | nextPage = (int) (this.file.size() / pageSize); |
| 314 | } |
| 315 | } |
| 316 | |
| 317 | public void insert(byte[] k, long v) throws Exception { |
| 318 | root.insert(k, v); |
| 319 | } |
| 320 | |
| 321 | public long search(byte[] k) throws Exception { |
| 322 | return root.search(k); |
| 323 | } |
| 324 | |
| 325 | Page allocate(boolean leaf) throws IOException { |
| 326 | Page page = new Page(nextPage++, leaf); |
| 327 | cache.put(page.number, page); |
| 328 | return page; |
| 329 | } |
| 330 | |
| 331 | Page getPage(int number) throws IOException { |
| 332 | Page page = cache.get(number); |
| 333 | if (page == null) { |
| 334 | page = new Page(number); |
| 335 | cache.put(number, page); |
| 336 | } |
| 337 | return page; |
| 338 | } |
| 339 | |
| 340 | public String toString() { |
| 341 | return root.toString(); |
| 342 | } |
| 343 | |
| 344 | public void close() throws IOException { |
| 345 | file.close(); |
| 346 | cache.clear(); |
| 347 | } |
| 348 | |
| 349 | public Iterator<byte[]> iterator() { |
| 350 | return root.iterator(); |
| 351 | } |
| 352 | } |