structuration-info.json

[
  {
    "file": "Cassandra-1011.docx",
    "title": "Cassandra-1011",
    "version": "cassandra-0.6.13",
    "Description": "Exception auto-bootstrapping two nodes nodes at the same time.\n3 machines in the cluster, and after starting the first node (which is the seed), the other two nodes are brought up at the same time. Then the following exception gets raised on the seed node. Looks like the seed node is assigning the same token to the subnodes at the same time",
    "logs": [],
    "stack_traces": [
      "java.lang.RuntimeException: Bootstrap Token collision between /10.0.0.2 and /10.0.0.3 (token Token (bytes[4c617374204d6967726174696f6e])\nat org.apache.cassandra.locator.TokenMetadata.addBootstrapToken(TokenMetadata.java:130)\nat org.apache.cassandra.service.StorageService.handleStateBootstrap(StorageService.java:548)\nat org.apache.cassandra.service.StorageService.onChange(StorageService.java:511)\nat org.apache.cassandra.gms.Gossiper.doNotifications(Gossiper.java:705)\nat org.apache.cassandra.gms.Gossiper.applyApplicationStateLocally(Gossiper.java:670)\nat org.apache.cassandra.gms.Gossiper.applyStateLocally(Gossiper.java:624)\nat org.apache.cassandra.gms.Gossiper$GossipDigestAck2VerbHandler.doVerb(Gossiper.java:1016)\nat org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:41)\nat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)\nat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)\nat java.lang.Thread.run(Thread.java:636)"
    ]
  },
  {
    "file": "Cassandra-1432.docx",
    "title": "Cassandra-1432",
    "version": "cassandra-0.6.5",
    "Description": "java.util.NoSuchElementException when returning a node to the cluster\n4 nodes in a cluster. One (34.27) of the nodes has been down (machine off, unclean shutdown) for about an hour, and not sure how many writes were going on. When this node was brought back, no error message is reported in the log",
    "logs": [
      "INFO [main] 2010-08-25 19:29:50,679 CommitLog.java (line 340) Recovery complete",
      "INFO [main] 2010-08-25 19:29:50,769 CommitLog.java (line 180) Log replay complete",
      "INFO [main] 2010-08-25 19:29:50,797 StorageService.java (line 342) Cassandra version: 0.7.0-beta1-SNAPSHOT",
      "INFO [main] 2010-08-25 19:29:50,797 StorageService.java (line 343) Thrift API version: 10.0.0",
      "INFO [main] 2010-08-25 19:29:50,813 SystemTable.java (line 240) Saved Token found: 85070591730234615865843651857942052864",
      "INFO [main] 2010-08-25 19:29:50,813 SystemTable.java (line 257) Saved ClusterName found: FOO",
      "INFO [main] 2010-08-25 19:29:50,813 SystemTable.java (line 272) Saved partitioner not found. Using org.apache.cassandra.dht.RandomPartitioner",
      "INFO [main] 2010-08-25 19:29:50,814 ColumnFamilyStore.java (line 422) switching in a fresh Memtable for LocationInfo at CommitLogContext(file='/local1/junkbox/cassandra/commitlog/CommitLog-1282721389770.log', position=41336)",
      "INFO [main] 2010-08-25 19:29:50,814 ColumnFamilyStore.java (line 706) Enqueuing flush of Memtable-LocationInfo@916236367(95 bytes, 2 operations)",
      "INFO [FLUSH-WRITER-POOL:1] 2010-08-25 19:29:50,815 Memtable.java (line 150) Writing Memtable-LocationInfo@916236367(95 bytes, 2 operations)",
      "INFO [FLUSH-WRITER-POOL:1] 2010-08-25 19:29:50,873 Memtable.java (line 157) Completed flushing /local1/junkbox/cassandra/data/system/LocationInfo-e-6-Data.db",
      "INFO [main] 2010-08-25 19:29:50,917 StorageService.java (line 374) Starting up server gossip",
      "INFO [main] 2010-08-25 19:29:51,093 ColumnFamilyStore.java (line 1239) Loaded 0 rows into the Super2 cache",
      "INFO [main] 2010-08-25 19:29:51,170 CassandraDaemon.java (line 153) Binding thrift service to /0.0.0.0:9160",
      "INFO [main] 2010-08-25 19:29:51,174 CassandraDaemon.java (line 167) Using TFramedTransport with a max frame size of 15728640 bytes.",
      "INFO [GOSSIP_STAGE:1] 2010-08-25 19:29:51,198 Gossiper.java (line 578) Node /192.168.34.28 is now part of the cluster",
      "INFO [GOSSIP_STAGE:1] 2010-08-25 19:29:51,199 Gossiper.java (line 578) Node /192.168.34.29 is now part of the cluster",
      "INFO [GOSSIP_STAGE:1] 2010-08-25 19:29:51,199 Gossiper.java (line 578) Node /192.168.34.26 is now part of the cluster",
      "INFO [main] 2010-08-25 19:29:51,204 CassandraDaemon.java (line 208) Listening for thrift clients...",
      "INFO [main] 2010-08-25 19:29:51,210 Mx4jTool.java (line 73) Will not load MX4J, mx4j-tools.jar is not in the classpath",
      "INFO [HINTED-HANDOFF-POOL:1] 2010-08-25 19:29:51,417 HintedHandOffManager.java (line 191) Started hinted handoff for endpoint /192.168.34.28",
      "INFO [GOSSIP_STAGE:1] 2010-08-25 19:29:51,417 Gossiper.java (line 570) InetAddress /192.168.34.28 is now UP",
      "INFO [HINTED-HANDOFF-POOL:1] 2010-08-25 19:29:51,418 HintedHandOffManager.java (line 247) Finished hinted handoff of 0 rows to endpoint /192.168.34.28",
      "INFO [HINTED-HANDOFF-POOL:1] 2010-08-25 19:29:51,855 HintedHandOffManager.java (line 191) Started hinted handoff for endpoint /192.168.34.29",
      "INFO [GOSSIP_STAGE:1] 2010-08-25 19:29:51,855 Gossiper.java (line 570) InetAddress /192.168.34.29 is now UP",
      "INFO [HINTED-HANDOFF-POOL:1] 2010-08-25 19:29:51,860 HintedHandOffManager.java (line 247) Finished hinted handoff of 0 rows to endpoint /192.168.34.29",
      "INFO [HINTED-HANDOFF-POOL:1] 2010-08-25 19:29:52,930 HintedHandOffManager.java (line 191) Started hinted handoff for endpoint /192.168.34.26",
      "INFO [GOSSIP_STAGE:1] 2010-08-25 19:29:52,930 Gossiper.java (line 570) InetAddress /192.168.34.26 is now UP",
      "INFO [HINTED-HANDOFF-POOL:1] 2010-08-25 19:29:52,930 HintedHandOffManager.java (line 247) Finished hinted handoff of 0 rows to endpoint /192.168.34.26",
      "INFO [GOSSIP_STAGE:1] 2010-08-25 19:29:51,199 Gossiper.java (line 584) Node /192.168.34.27 has restarted, now UP again",
      "INFO [HINTED-HANDOFF-POOL:1] 2010-08-25 19:29:51,200 HintedHandOffManager.java (line 191) Started hinted handoff for endpoint /192.168.34.27",
      "INFO [GOSSIP_STAGE:1] 2010-08-25 19:29:51,201 StorageService.java (line 636) Node /192.168.34.27 state jump to normal",
      "INFO [GOSSIP_STAGE:1] 2010-08-25 19:29:51,201 StorageService.java (line 643) Will not change my token ownership to /192.168.34.27",
      "ERROR [HINTED-HANDOFF-POOL:1] 2010-08-25 19:29:51,640 CassandraDaemon.java (line 82) Uncaught exception in thread Thread[HINTED-HANDOFF-POOL:1,5,main]",
      "INFO [manual-repair-fe7c5abb-bb0a-4415-aa75-0d72ba4e7f1b] 2010-08-25 19:49:24,194 AntiEntropyService.java (line 803) Waiting for repair requests to: []"
    ],
    "stack_traces": [
      "java.util.concurrent.ExecutionException: java.lang.RuntimeException: java.util.NoSuchElementException\nat java.util.concurrent.FutureTask$Sync.innerGet(FutureTask.java:222)\nat java.util.concurrent.FutureTask.get(FutureTask.java:83)\nat org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor.afterExecute(DebuggableThreadPoolExecutor.java:87)\nat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:888)\nat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\nat java.lang.Thread.run(Thread.java:619)\nCaused by: java.lang.RuntimeException: java.util.NoSuchElementException\nat orgapache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:34)\nat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)\nat java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)\nat java.util.concurrent.FutureTask.run(FutureTask.java:138)\nat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\n... 2 more\nCaused by: java.util.NoSuchElementException\nat java.util.concurrent.ConcurrentSkipListMap.lastKey(ConcurrentSkipListMap.java:1981)\nat java.util.concurrent.ConcurrentSkipListMap$KeySet.last(ConcurrentSkipListMap.java:2331)\nat org.apache.cassandra.db.HintedHandOffManager.sendMessage(HintedHandOffManager.java:121)\nat org.apache.cassandra.db.HintedHandOffManager.deliverHintsToEndpoint(HintedHandOffManager.java:218)\nat org.apache.cassandra.db.HintedHandOffManager.access$000(HintedHandOffManager.java:78)\nat org.apache.cassandra.db.HintedHandOffManager$1.runMayThrow(HintedHandOffManager.java:296)\nat org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:30)\n... 6 more"
    ]
  },
  {
    "file": "Cassandra-1463.docx",
    "title": "Cassandra-1463",
    "version": "cassandra-0.6.5",
    "Description": "Failed bootstrap can cause NPE in batch_mutate on every node, taking down the entire cluster\nIn adding a node to the cluster, the bootstrap failed (still investigating the cause). An hour later, the entire cluster failed, preventing any writes from being accepted.",
    "logs": [
      "INFO [Timer-0] 2010-09-03 12:23:33,282 Gossiper.java (line 402) FatClient /10.251.243.191 has been silent for 3600000ms, removing from gossip",
      "ERROR [Timer-0] 2010-09-03 12:23:33,318 Gossiper.java (line 99) Gossip error",
      "ERROR Cassandra.java (line 1659) Internal error processing batch_mutate",
      "INFO 05:26:36,245 FatClient /10.179.65.102 has been silent for 3600000ms, removing from gossip",
      "ERROR 05:26:36,247 Uncaught exception in thread Thread[Timer-0,5,main]"
    ],
    "stack_traces": [
      "java.util.ConcurrentModificationException\nat java.util.Hashtable$Enumerator.next(Hashtable.java:1048)\nat org.apache.cassandra.gms.Gossiper.doStatusCheck(Gossiper.java:383)\nat org.apache.cassandra.gms.Gossiper$GossipTimerTask.run(Gossiper.java:93)\nat java.util.TimerThread.mainLoop(Timer.java:534)\nat java.util.TimerThread.run(Timer.java:484)",
      "java.lang.NullPointerException\nat org.apache.cassandra.gms.FailureDetector.isAlive(FailureDetector.java:135)\nat org.apache.cassandra.locator.AbstractReplicationStrategy.getHintedEndpoints(AbstractReplicationStrategy.java:85)\nat org.apache.cassandra.service.StorageProxy.mutateBlocking(StorageProxy.java:204)\nat org.apache.cassandra.thrift.CassandraServer.batch_mutate(CassandraServer.java:415)\nat org.apache.cassandra.thrift.Cassandra$Processor$batch_mutate.process(Cassandra.java:1651)\nat org.apache.cassandra.thrift.Cassandra$Processor.process(Cassandra.java:1166)\nat org.apache.cassandra.thrift.CustomTThreadPoolServer$WorkerProcess.run(CustomTThreadPoolServer.java:167)\nat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)\nat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)\nat java.lang.Thread.run(Thread.java:636)",
      "java.lang.AssertionError\nat org.apache.cassandra.locator.TokenMetadata.removeEndpoint(TokenMetadata.java:192)\nat org.apache.cassandra.service.StorageService.onRemove(StorageService.java:879)\nat org.apache.cassandra.gms.Gossiper.removeEndPoint(Gossiper.java:221)\nat org.apache.cassandra.gms.Gossiper.doStatusCheck(Gossiper.java:407)\nat org.apache.cassandra.gms.Gossiper$GossipTimerTask.run(Gossiper.java:93)\nat java.util.TimerThread.mainLoop(Timer.java:534)\nat java.util.TimerThread.run(Timer.java:484)"
    ]
  },
  {
    "file": "Cassandra-2525.docx",
    "title": "Cassandra-2525",
    "version": "cassandra-0.8.0-beta1",
    "Description": "CQL: create keyspace does not the replication factor argument and allows invalid sql to pass thru",
    "logs": [
      "ERROR 01:29:26,989 Exception encountered during startup."
    ],
    "stack_traces": [
      "java.lang.RuntimeException: org.apache.cassandra.config.ConfigurationException: SimpleStrategy requires a replication_factor strategy option.\n\t\t\t\t\tat org.apache.cassandra.db.Table.<init>(Table.java:278)\n\t\t\t\t\tat org.apache.cassandra.db.Table.open(Table.java:110)\n\t\t\t\t\tat org.apache.cassandra.service.AbstractCassandraDaemon.setup(AbstractCassandraDaemon.java:160)\n\t\t\t\t\tat org.apache.cassandra.service.AbstractCassandraDaemon.activate(AbstractCassandraDaemon.java:314)\n\t\t\t\t\tat org.apache.cassandra.thrift.CassandraDaemon.main(CassandraDaemon.java:80)\nCaused by: org.apache.cassandra.config.ConfigurationException: SimpleStrategy requires a replication_factor strategy option.\n\t\t\t\t\tat org.apache.cassandra.locator.SimpleStrategy.validateOptions(SimpleStrategy.java:79)\n\t\t\t\t\tat org.apache.cassandra.locator.AbstractReplicationStrategy.createReplicationStrategy(AbstractReplicationStrategy.java:262)\n\t\t\t\t\tat org.apache.cassandra.db.Table.createReplicationStrategy(Table.java:328)\n\t\t\t\t\tat org.apache.cassandra.db.Table.<init>(Table.java:274)\n\t\t\t\t\t... 4 more"
    ]
  },
  {
    "file": "Cassandra-2752.docx",
    "title": "Cassandra-2752",
    "version": "cassandra-0.8.0",
    "Description": "Repair fails with java.io.EOFException",
    "logs": [
      "INFO [AntiEntropyStage:1] 2011-06-09 19:02:47,999 AntiEntropyService.java (line 234) Queueing comparison #<Differencer #<TreeRequest manual-repair-0c17c5f9-583f-4a31-a6d4-a9e7306fb46e, /1.10.42.82, (JP,XXX), (Token(bytes[6e]),Token(bytes[313039]))>>",
      "INFO [AntiEntropyStage:1] 2011-06-09 19:02:48,026 AntiEntropyService.java (line 468) Endpoints somewhere/1.10.42.81 and /1.10.42.82 have 2 range(s) out of sync for (JP,XXX) on (Token(bytes[6e]),Token(bytes[313039]))",
      "INFO [AntiEntropyStage:1] 2011-06-09 19:02:48,026 AntiEntropyService.java (line 485) Performing streaming repair of 2 ranges for #<TreeRequest manual-repair-0c17c5f9-583f-4a31-a6d4-a9e7306fb46e, /z, (JP,XXX), (Token(bytes[6e]),Token(bytes[313039]))>",
      "INFO [AntiEntropyStage:1] 2011-06-09 19:02:48,030 StreamOut.java (line 173) Stream context metadata[/data/cassandra/node0/data/JP/XXX-g-3-Data.db sections=1 progress=0/36592 - 0%], 1 sstables.",
      "INFO [AntiEntropyStage:1] 2011-06-09 19:02:48,031 StreamOutSession.java (line 174) Streaming to /1.10.42.82",
      "ERROR [CompactionExecutor:9] 2011-06-09 19:02:48,970 AbstractCassandraDaemon.java (line 113) Fatal exception in thread Thread[CompactionExecutor:9,1,main]",
      "ERROR [CompactionExecutor:12] 2011-06-09 19:02:48,051 AbstractCassandraDaemon.java (line 113) Fatal exception in thread Thread[CompactionExecutor:12,1,main]",
      "ERROR [Thread-132] 2011-06-09 19:02:48,051 AbstractCassandraDaemon.java (line 113) Fatal exception in thread Thread[Thread-132,5,main]"
    ],
    "stack_traces": [
      "java.io.EOFException\nat java.io.RandomAccessFile.readInt(RandomAccessFile.java:725)\nat org.apache.cassandra.io.sstable.SSTableWriter$RowIndexer.doIndexing(SSTableWriter.java:457)\nat org.apache.cassandra.io.sstable.SSTableWriter$RowIndexer.index(SSTableWriter.java:364)\nat org.apache.cassandra.io.sstable.SSTableWriter$Builder.build(SSTableWriter.java:315)\nat org.apache.cassandra.db.CompactionManager$9.call(CompactionManager.java:1099)\nat org.apache.cassandra.db.CompactionManager$9.call(CompactionManager.java:1090)\nat java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)\nat java.util.concurrent.FutureTask.run(FutureTask.java:138)\nat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\nat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\nat java.lang.Thread.run(Thread.java:662)",
      "java.lang.RuntimeException: java.util.concurrent.ExecutionException: java.io.EOFException\nat org.apache.cassandra.streaming.StreamInSession.closeIfFinished(StreamInSession.java:152)\nat org.apache.cassandra.streaming.IncomingStreamReader.read(IncomingStreamReader.java:63)\nat org.apache.cassandra.net.IncomingTcpConnection.stream(IncomingTcpConnection.java:155)\nat org.apache.cassandra.net.IncomingTcpConnection.run(IncomingTcpConnection.java:93)\nCaused by: java.util.concurrent.ExecutionException: java.io.EOFException\nat java.util.concurrent.FutureTask$Sync.innerGet(FutureTask.java:222)\nat java.util.concurrent.FutureTask.get(FutureTask.java:83)\nat org.apache.cassandra.streaming.StreamInSession.closeIfFinished(StreamInSession.java:136)\n... 3 more\nCaused by: java.io.EOFException\nat java.io.RandomAccessFile.readInt(RandomAccessFile.java:725)\nat org.apache.cassandra.io.sstable.SSTableWriter$RowIndexer.doIndexing(SSTableWriter.java:457)\nat org.apache.cassandra.io.sstable.SSTableWriter$RowIndexer.index(SSTableWriter.java:364)\nat org.apache.cassandra.io.sstable.SSTableWriter$Builder.build(SSTableWriter.java:315)\nat org.apache.cassandra.db.CompactionManager$9.call(CompactionManager.java:1099)\nat org.apache.cassandra.db.CompactionManager$9.call(CompactionManager.java:1090)\nat java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)\nat java.util.concurrent.FutureTask.run(FutureTask.java:138)\nat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\nat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\nat java.lang.Thread.run(Thread.java:662)"
    ]
  },
  {
    "file": "Cassandra-2773.docx",
    "title": "Cassandra-2773",
    "version": "cassandra-0.7.6",
    "Description": "Index manager cannot support deleting and inserting into a row in the same mutation",
    "logs": [
      "INFO 17:20:39,318 reading saved cache /var/lib/cassandra/saved_caches/SOCommContent_S1-AgentPropsMirror-KeyCache",
      "INFO 17:20:39,509 reading saved cache /var/lib/cassandra/saved_caches/SOCommContent_S1-AgentProps-KeyCache",
      "INFO 17:20:39,510 Opening /var/lib/cassandra/data/SOCommContent_S1/AgentProps-f-7",
      "INFO 17:20:39,511 Opening /var/lib/cassandra/data/SOCommContent_S1/AgentProps-f-6",
      "INFO 17:20:39,511 Opening /var/lib/cassandra/data/SOCommContent_S1/AgentProps-f-5",
      "INFO 17:20:39,522 Creating new commitlog segment /var/lib/cassandra/commitlog/CommitLog-1308864039522.log",
      "INFO 17:20:39,530 Replaying /var/lib/cassandra/commitlog/CommitLog-1308164298316.log, /var/lib/cassandra/commitlog/CommitLog-1308632419965.log, /var/lib/cassandra/commitlog/CommitLog-1308841938215.log, /var/lib/cassandra/commitlog/CommitLog-1308856663370.log, /var/lib/cassandra/commitlog/CommitLog-1308864021483.log",
      "INFO 17:20:41,190 Finished reading /var/lib/cassandra/commitlog/CommitLog-1308164298316.log",
      "INFO 17:20:41,343 Finished reading /var/lib/cassandra/commitlog/CommitLog-1308632419965.log",
      "ERROR 17:20:41,385 Fatal exception in thread Thread[MutationStage:5,5,main]",
      "ERROR 17:20:41,389 Fatal exception in thread Thread[MutationStage:58,5,main]"
    ],
    "stack_traces": [
      "java.lang.RuntimeException: java.lang.UnsupportedOperationException: Index manager cannot support deleting and inserting into a row in the same mutation\n\t\t\t\t\tat org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:34)\n\t\t\t\t\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)\n\t\t\t\t\tat java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)\n\t\t\t\t\tat java.util.concurrent.FutureTask.run(FutureTask.java:138)\n\t\t\t\t\tat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\n\t\t\t\t\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\n\t\t\t\t\tat java.lang.Thread.run(Thread.java:619)\nCaused by: java.lang.UnsupportedOperationException: Index manager cannot support deleting and inserting into a row in the same mutation\n\t\t\t\t\tat org.apache.cassandra.db.Table.ignoreObsoleteMutations(Table.java:431)\n\t\t\t\t\tat org.apache.cassandra.db.Table.apply(Table.java:387)\n\t\t\t\t\tat org.apache.cassandra.db.commitlog.CommitLog$2.runMayThrow(CommitLog.java:300)\n\t\t\t\t\tat org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:30)\n\t\t\t\t\t... 6 more"
    ]
  },
  {
    "file": "Cassandra-2792.docx",
    "title": "Cassandra-2792",
    "version": "cassandra-0.7.6",
    "Description": "Bootstrapping node stalls. Bootstrapper thinks it is still streaming some sstables. The source nodes do not. Caused by IllegalStateException on source nodes.",
    "logs": [
      "INFO [StreamStage:1] 2011-06-17 22:27:05,924 StreamOut.java (line 126) Beginning transfer to /192.168.1.9",
      "INFO [StreamStage:1] 2011-06-17 22:27:05,925 StreamOut.java (line 100) Flushing memtables for FMM_Studio...",
      "INFO [StreamStage:1] 2011-06-17 22:27:06,004 StreamOut.java (line 173) Stream context metadata [/var/opt/cassandra/data/FMM_Studio/Classes-f-107-Data.db sections=1 progress=0/1585378 - 0%, /var/opt/cassandra/data/FMM_Studio/PartsData-f-100-Data.db sections=1 progress=0/76453 - 0%, /var/opt/cassandra/data/FMM_Studio/PartsData-f-98-Data.db sections=1 progress=0/4309514 - 0%, /var/opt/cassandra/data/FMM_Studio/PartsData-f-99-Data.db sections=1 progress=0/90475 - 0%], 11 sstables.",
      "INFO [StreamStage:1] 2011-06-17 22:27:06,005 StreamOutSession.java (line 174) Streaming to /192.168.1.9",
      "INFO [StreamStage:1] 2011-06-17 22:27:06,006 StreamOut.java (line 126) Beginning transfer to /192.168.1.9",
      "INFO [StreamStage:1] 2011-06-17 22:27:06,007 StreamOut.java (line 100) Flushing memtables for FightMyMonster...",
      "INFO [StreamStage:1] 2011-06-17 22:27:06,007 ColumnFamilyStore.java (line 1065) Enqueuing flush of Memtable-MonsterMarket_1@1054909557(338 bytes, 24 operations)",
      "INFO [StreamStage:1] 2011-06-17 22:27:06,007 ColumnFamilyStore.java (line 1065) Enqueuing flush of Memtable-UserFights@239934867(1124836 bytes, 965 operations)",
      "INFO [FlushWriter:409] 2011-06-17 22:27:06,007 Memtable.java (line 157) Writing Memtable-MonsterMarket_1@1054909557(338 bytes, 24 operations)",
      "INFO [StreamStage:1] 2011-06-17 22:27:06,007 ColumnFamilyStore.java (line 1065) Enqueuing flush of Memtable-Users_CisIndex@1758504250(242 bytes, 8 operations)",
      "INFO [StreamStage:1] 2011-06-17 22:27:06,008 ColumnFamilyStore.java (line 1065) Enqueuing flush of Memtable-Tribes@1510979736(18318 bytes, 703 operations)",
      "INFO [StreamStage:1] 2011-06-17 22:27:06,008 ColumnFamilyStore.java (line 1065) Enqueuing flush of Memtable-ColumnViews_TimeUUID@864545260(2073 bytes, 63 operations)",
      "INFO [StreamStage:1] 2011-06-17 22:27:06,008 ColumnFamilyStore.java (line 1065) Enqueuing flush of Memtable-MonsterMarket_0@537829218(2600 bytes, 129 operations)",
      "INFO [FlushWriter:409] 2011-06-17 22:27:06,069 Memtable.java (line 172) Completed flushing /var/opt/cassandra/data/FightMyMonster/MonsterMarket_1-f-3799-Data.db (1774 bytes)",
      "INFO [FlushWriter:409] 2011-06-17 22:27:06,069 Memtable.java (line 157) Writing Memtable-UserFights@239934867(1124836 bytes, 965 operations)",
      "INFO [StreamStage:1] 2011-06-17 22:27:06,070 ColumnFamilyStore.java (line 1065) Enqueuing flush of Memtable-UserSigninLog@1692186117(4043 bytes, 137 operations)",
      "INFO [FlushWriter:409] 2011-06-17 22:27:06,161 Memtable.java (line 172) Completed flushing /var/opt/cassandra/data/FightMyMonster/UserFights-f-8192-Data.db (1179202 bytes)",
      "INFO [FlushWriter:409] 2011-06-17 22:27:06,161 Memtable.java (line 157) Writing Memtable-Users_CisIndex@1758504250(242 bytes, 8 operations)",
      "INFO [CompactionExecutor:1] 2011-06-17 22:27:06,161 CompactionManager.java (line 395) Compacting [SSTableReader(path='/var/opt/cassandra/data/FightMyMonster/UserFights-f-8189-Data.db'),SSTableReader(path='/var/opt/cassandra/data/FightMyMonster/UserFights-f-8190-Data.db'),SSTableReader(path='/var/opt/cassandra/data/FightMyMonster/UserFights-f-8191-Data.db'),SSTableReader(path='/var/opt/cassandra/data/FightMyMonster/UserFights-f-8192-Data.db')]",
      "INFO [StreamStage:1] 2011-06-17 22:27:06,162 ColumnFamilyStore.java (line 1065) Enqueuing flush of Memtable-TribeFights@321579649(138 bytes, 3 operations)",
      "ERROR [MiscStage:1] 2011-06-17 22:27:06,168 AbstractCassandraDaemon.java (line 114) Fatal exception in thread Thread[MiscStage:1,5,main]"
    ],
    "stack_traces": [
      "java.lang.IllegalStateException: target reports current file is /var/opt/cassandra/data/FMM_Studio/Classes-f-107-Data.db but is null\nat org.apache.cassandra.streaming.StreamOutSession.validateCurrentFile(StreamOutSession.java:166)\nat org.apache.cassandra.streaming.StreamReplyVerbHandler.doVerb(StreamReplyVerbHandler.java:58)\nat org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:72)\nat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\nat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\nat java.lang.Thread.run(Thread.java:619)"
    ]
  },
  {
    "file": "Cassandra-2825.docx",
    "title": "Cassandra-2825",
    "version": "cassandra-0.8.2",
    "Description": "Auto bootstrapping the 4th node in a 4 node cluster doesn't work, when no token explicitly assigned in config",
    "logs": [
      "INFO [GossipStage:1] 2011-06-24 16:40:41,947 Gossiper.java (line 638) Node /10.171.47.226 is now part of the cluster",
      "INFO [GossipStage:1] 2011-06-24 16:40:41,947 Gossiper.java (line 606) InetAddress /10.171.47.226 is now UP",
      "INFO [GossipStage:1] 2011-06-24 16:42:09,432 StorageService.java (line 769) Nodes /10.171.47.226 and /10.171.55.77 have the same token 61078635599166706937511052402724559481. /10.171.47.226 is the new owner",
      "WARN [GossipStage:1] 2011-06-24 16:42:09,432 TokenMetadata.java (line 120) Token 61078635599166706937511052402724559481 changing ownership from /10.171.55.77 to /10.171.47.226"
    ],
    "stack_traces": []
  },
  {
    "file": "Cassandra-2946.docx",
    "title": "Cassandra-2946",
    "version": "cassandra-0.8.2",
    "Description": "HintedHandoff fails with could not reach schema agreement",
    "logs": [
      "ERROR [HintedHandoff:1] 2011-07-25 17:19:14,729 AbstractCassandraDaemon.java (line 139) Fatal exception in thread Thread[HintedHandoff:1,1,main]",
      "DEBUG [HintedHandoff:1] 2011-07-26 11:22:35,526 HintedHandOffManager.java (line 300) Checking remote schema before delivering hints",
      "DEBUG [pool-2-thread-1] 2011-07-26 11:22:44,965 CassandraServer.java (line 1123) checking schema agreement",
      "DEBUG [pool-2-thread-1] 2011-07-26 11:22:44,969 StorageProxy.java (line 823) Schemas are in agreement.",
      "ERROR [HintedHandoff:1] 2011-07-26 11:23:36,788 AbstractCassandraDaemon.java (line 138) Fatal exception in thread Thread[HintedHandoff:1,1,main]"
    ],
    "stack_traces": [
      "java.lang.RuntimeException: java.lang.RuntimeException: Could not reach schema agreement with /10.2.129.9 in 60000ms\nat org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:34)\nat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)\nat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)\nat java.lang.Thread.run(Thread.java:636)\nCaused by: java.lang.RuntimeException: Could not reach schema agreement with /10.2.129.9 in 60000ms\nat org.apache.cassandra.db.HintedHandOffManager.waitForSchemaAgreement(HintedHandOffManager.java:290)\nat org.apache.cassandra.db.HintedHandOffManager.deliverHintsToEndpoint(HintedHandOffManager.java:301)\nat org.apache.cassandra.db.HintedHandOffManager.access$100(HintedHandOffManager.java:89)\nat org.apache.cassandra.db.HintedHandOffManager$2.runMayThrow(HintedHandOffManager.java:394)\nat org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:30)\n... 3 more"
    ]
  },
  {
    "file": "Cassandra-2948.docx",
    "title": "Cassandra-2948",
    "version": "cassandra-0.8.2",
    "Description": "Nodetool move fails to stream out data from moved node to new endpoint.",
    "logs": [
      "INFO [RMI TCP Connection(6)-10.2.129.41] 2011-07-26 16:29:17,075 StorageService.java (line 1878) Moving miles/10.2.129.41 from Token(bytes[00]) to Token(bytes[07]).",
      "DEBUG [RMI TCP Connection(6)-10.2.129.41] 2011-07-26 16:29:17,080 StorageService.java (line 1941) Table ks: work map {/10.2.129.16=[(Token(bytes[04]),Token(bytes[07])]]}.",
      "INFO [RMI TCP Connection(6)-10.2.129.41] 2011-07-26 16:29:17,080 StorageService.java (line 1946) Sleeping 30000 ms before start streaming/fetching ranges.",
      "INFO [RMI TCP Connection(6)-10.2.129.41] 2011-07-26 16:29:46,728 StorageService.java (line 522) Moving: fetching new ranges and streaming old ranges",
      "DEBUG [RMI TCP Connection(6)-10.2.129.41] 2011-07-26 16:29:46,728 StorageService.java (line 1960) [Move->STREAMING] Work Map: {ks={(Token(bytes[0c]),Token(bytes[00])]=[miles/10.2.129.41]}}",
      "DEBUG [RMI TCP Connection(6)-10.2.129.41] 2011-07-26 16:29:46,729 StorageService.java (line 1965) [Move->FETCHING] Work Map: {ks={/10.2.129.16=[(Token(bytes[04]),Token(bytes[07])]]}}",
      "DEBUG [RMI TCP Connection(6)-10.2.129.41] 2011-07-26 16:29:46,730 StorageService.java (line 2411) Requesting from /10.2.129.16 ranges (Token(bytes[04]),Token(bytes[07])]",
      "INFO [StreamStage:1] 2011-07-26 16:29:46,737 StreamOut.java (line 90) Beginning transfer to miles/10.2.129.41",
      "DEBUG [StreamStage:1] 2011-07-26 16:29:46,737 StreamOut.java (line 91) Ranges are (Token(bytes[0c]),Token(bytes[00])]"
    ],
    "stack_traces": []
  },
  {
    "file": "Cassandra-3156.docx",
    "title": "Cassandra-3156",
    "version": "cassandra-1.0.0-rc2",
    "Description": "Assertion error in RowRepairResolver",
    "logs": [
      "DEBUG 03:15:59,866 Processing response on a callback from 3840@/10.179.64.227",
      "DEBUG 03:15:59,866 Preprocessed data response",
      "DEBUG 03:15:59,866 Processing response on a callback from 3841@/10.179.111.137",
      "DEBUG 03:15:59,866 Preprocessed digest response",
      "DEBUG 03:15:59,865 Processing response on a callback from 3837@/10.179.111.137",
      "DEBUG 03:15:59,865 Preprocessed data response",
      "DEBUG 03:15:59,867 Preprocessed digest response",
      "DEBUG 03:15:59,867 resolving 2 responses",
      "ERROR 03:15:59,866 Fatal exception in thread Thread[ReadRepairStage:526,5,main]",
      "ERROR 03:15:59,866 Fatal exception in thread Thread[ReadRepairStage:525,5,main]",
      "ERROR 03:15:59,867 Fatal exception in thread Thread[ReadRepairStage:528,5,main]"
    ],
    "stack_traces": [
      "java.lang.AssertionError\nat org.apache.cassandra.service.RowRepairResolver.resolve(RowRepairResolver.java:77)\nat org.apache.cassandra.service.AsyncRepairCallback$1.runMayThrow(AsyncRepairCallback.java:54)\nat org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:30)\nat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\nat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\nat java.lang.Thread.run(Thread.java:662)"
    ]
  },
  {
    "file": "Cassandra-3369.docx",
    "title": "Cassandra-3369",
    "version": "cassandra-1.0.0-rc2",
    "Description": "AssertionError when adding a node and doing repair, repair hangs",
    "logs": [
      "INFO [GossipStage:1] 2011-10-16 18:15:46,837 Gossiper.java (line 737) Node /127.0.0.2 is now part of the cluster",
      "INFO [GossipStage:1] 2011-10-16 18:15:46,838 Gossiper.java (line 703) InetAddress /127.0.0.2 is now UP",
      "INFO [RMI TCP Connection(22)-127.0.1.1] 2011-10-16 18:17:01,221 StorageService.java (line 1655) Starting repair command #1, repairing 2 ranges.",
      "INFO [AntiEntropySessions:1] 2011-10-16 18:17:01,229 AntiEntropyService.java (line 644) [repair #4768dfb0-f812-11e0-0000-fe8ebeead9fb] new session: will sync /127.0.0.1, /127.0.0.2 on range (Token(bytes[4d6ccfeaa8bb59551751a2816fde9343]),Token(bytes[63e5b6995466cd3221cba16646ae19ed])] for Test.[Response]",
      "INFO [AntiEntropySessions:1] 2011-10-16 18:17:01,231 AntiEntropyService.java (line 793) [repair #4768dfb0-f812-11e0-0000-fe8ebeead9fb] requests for merkle tree sent for Response (to [/127.0.0.1, /127.0.0.2])",
      "INFO [ValidationExecutor:1] 2011-10-16 18:17:01,240 ColumnFamilyStore.java (line 664) Enqueuing flush of Memtable-Response@1405292188(108646086/135807607 serialized/live bytes, 5298 ops)",
      "INFO [FlushWriter:6] 2011-10-16 18:17:01,240 Memtable.java (line 237) Writing Memtable-Response@1405292188(108646086/135807607 serialized/live bytes, 5298 ops)",
      "INFO [AntiEntropyStage:1] 2011-10-16 18:17:02,054 AntiEntropyService.java (line 175) [repair #4768dfb0-f812-11e0-0000-fe8ebeead9fb] Received merkle tree for Response from /127.0.0.2",
      "INFO [FlushWriter:6] 2011-10-16 18:17:08,540 Memtable.java (line 273) Completed flushing /home/cspriegel/Development/cassandra1/data/Test/Response-h-3-Data.db (108914411 bytes)",
      "INFO [AntiEntropyStage:1] 2011-10-16 18:17:13,638 AntiEntropyService.java (line 175) [repair #4768dfb0-f812-11e0-0000-fe8ebeead9fb] Received merkle tree for Response from /127.0.0.1",
      "INFO [AntiEntropySessions:2] 2011-10-16 18:17:13,640 AntiEntropyService.java (line 644) [repair #4ececb70-f812-11e0-0000-fe8ebeead9fb] new session: will sync /127.0.0.1, /127.0.0.2 on range (Token(bytes[63e5b6995466cd3221cba16646ae19ed]),Token(bytes[4d6ccfeaa8bb59551751a2816fde9343])] for Test.[Response]",
      "INFO [AntiEntropySessions:2] 2011-10-16 18:17:13,640 AntiEntropyService.java (line 793) [repair #4ececb70-f812-11e0-0000-fe8ebeead9fb] requests for merkle tree sent for Response (to [/127.0.0.1, /127.0.0.2])",
      "INFO [AntiEntropyStage:1] 2011-10-16 18:17:13,704 AntiEntropyService.java (line 886) [repair #4768dfb0-f812-11e0-0000-fe8ebeead9fb] Endpoints /127.0.0.2 and /127.0.0.1 have 6 range(s) out of sync for Response",
      "INFO [AntiEntropyStage:1] 2011-10-16 18:17:13,706 StreamingRepairTask.java (line 117) [streaming task #4ed904a0-f812-11e0-0000-fe8ebeead9fb] Performing streaming repair of 6 ranges with /127.0.0.2",
      "INFO [AntiEntropyStage:1] 2011-10-16 18:17:13,735 StreamOut.java (line 160) Stream context metadata [/home/cspriegel/Development/cassandra1/data/Test/Response-h-1-Data.db sections=6 progress=0/34987700 - 0%, /home/cspriegel/Development/cassandra1/data/Test/Response-h-2-Data.db sections=4 progress=0/41162000 - 0%, /home/cspriegel/Development/cassandra1/data/Test/Response-h-3-Data.db sections=1 progress=0/10290500 - 0%], 3 sstables.",
      "INFO [AntiEntropyStage:1] 2011-10-16 18:17:13,737 StreamOutSession.java (line 203) Streaming to /127.0.0.2",
      "ERROR [ValidationExecutor:1] 2011-10-16 18:17:13,764 AbstractCassandraDaemon.java (line 133) Fatal exception in thread Thread[ValidationExecutor:1,5,main]"
    ],
    "stack_traces": [
      "java.lang.AssertionError\nat org.apache.cassandra.service.AntiEntropyService$Validator.prepare(AntiEntropyService.java:283)\nat org.apache.cassandra.db.compaction.CompactionManager.doValidationCompaction(CompactionManager.java:825)\nat org.apache.cassandra.db.compaction.CompactionManager.access$600(CompactionManager.java:63)\nat org.apache.cassandra.db.compaction.CompactionManager$6.call(CompactionManager.java:432)\nat java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)\nat java.util.concurrent.FutureTask.run(FutureTask.java:138)\nat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\nat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\nat java.lang.Thread.run(Thread.java:662)"
    ]
  },
  {
    "file": "Cassandra-3400.docx",
    "title": "Cassandra-3400",
    "version": "cassandra-0.8.7",
    "Description": "ConcurrentModificationException during nodetool repair",
    "logs": [
      "ERROR [AntiEntropySessions:12] 2011-10-24 11:17:52,154 AbstractCassandraDaemon.java (line 139) Fatal exception in thread Thread[AntiEntropySessions:12,5,RMI Runtime]"
    ],
    "stack_traces": [
      "java.lang.RuntimeException: java.util.ConcurrentModificationException\nat org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:34)\nat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)\nat java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)\nat java.util.concurrent.FutureTask.run(FutureTask.java:138)\nat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\nat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\nat java.lang.Thread.run(Thread.java:619)\nCaused by: java.util.ConcurrentModificationException\nat java.util.HashMap$HashIterator.nextEntry(HashMap.java:793)\nat java.util.HashMap$KeyIterator.next(HashMap.java:828)\nat org.apache.cassandra.service.AntiEntropyService$RepairSession$RepairJob.sendTreeRequests(AntiEntropyService.java:784)\nat org.apache.cassandra.service.AntiEntropyService$RepairSession.runMayThrow(AntiEntropyService.java:680)\nat org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:30)\n... 6 more"
    ]
  },
  {
    "file": "Cassandra-3548.docx",
    "title": "Cassandra-3548",
    "version": "cassandra-1.0.5",
    "Description": "NPE in AntiEntropyService$RepairSession.completed()",
    "logs": [
      "INFO [AntiEntropyStage:1] 2011-11-28 06:22:56,225 StreamingRepairTask.java (line 136) [streaming task #69187510-1989-11e1-0000-5ff37d368cb6] Forwarding streaming repair of 8602 ranges to /10.6.130.70 (to be streamed with /10.37.114.10)",
      "INFO [AntiEntropyStage:66] 2011-11-29 11:20:57,109 StreamingRepairTask.java (line 253) [streaming task #69187510-1989-11e1-0000-5ff37d368cb6] task succeeded",
      "ERROR [AntiEntropyStage:66] 2011-11-29 11:20:57,109 AbstractCassandraDaemon.java (line 133) Fatal exception in thread Thread[AntiEntropyStage:66,5,main]",
      "ERROR [AntiEntropySessions:1] 2011-11-28 19:39:52,507 AntiEntropyService.java (line 688) [repair #2bf19860-197f-11e1-0000-5ff37d368cb6] session completed with the following error"
    ],
    "stack_traces": [
      "java.lang.NullPointerException\n\t\tat org.apache.cassandra.service.AntiEntropyService$RepairSession.completed(AntiEntropyService.java:712)\n\t\tat org.apache.cassandra.service.AntiEntropyService$RepairSession$Differencer$1.run(AntiEntropyService.java:912)\n\t\tat org.apache.cassandra.streaming.StreamingRepairTask$2.run(StreamingRepairTask.java:186)\n\t\tat org.apache.cassandra.streaming.StreamingRepairTask$StreamingRepairResponse.doVerb(StreamingRepairTask.java:255)\n\t\tat org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:59)\n\t\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)\n\t\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)\n\t\tat java.lang.Thread.run(Thread.java:679)",
      "java.io.IOException: Endpoint /10.29.60.10 died\n\t\tat org.apache.cassandra.service.AntiEntropyService$RepairSession.failedNode(AntiEntropyService.java:725)\n\t\tat org.apache.cassandra.service.AntiEntropyService$RepairSession.convict(AntiEntropyService.java:762)\n\t\tat org.apache.cassandra.gms.FailureDetector.interpret(FailureDetector.java:192)\n\t\tat org.apache.cassandra.gms.Gossiper.doStatusCheck(Gossiper.java:559)\n\t\tat org.apache.cassandra.gms.Gossiper.access$700(Gossiper.java:62)\n\t\tat org.apache.cassandra.gms.Gossiper$GossipTask.run(Gossiper.java:167)\n\t\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)\n\t\tat java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:351)\n\t\tat java.util.concurrent.FutureTask.runAndReset(FutureTask.java:178)\n\t\tat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:165)\n\t\tat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:267)\n\t\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)\n\t\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)\n\t\tat java.lang.Thread.run(Thread.java:679)"
    ]
  },
  {
    "file": "Cassandra-3712.docx",
    "title": "Cassandra-3712",
    "version": "cassandra-1.0.7",
    "Description": "Can't cleanup after I moved a token",
    "logs": [
      "INFO [CompactionExecutor:260] 2012-01-09 14:08:41,716 CompactionManager.java (line 702) Cleaning up SSTableReader(path='/ke/cassandra/data/kev3/URLs-hc-457-Data.db')",
      "INFO [OptionalTasks:1] 2012-01-09 14:08:47,220 MeteredFlusher.java (line 62) flushing high-traffic column family CFS(Keyspace='kev3', ColumnFamily='URLs') (estimated 156787206 bytes)",
      "INFO [OptionalTasks:1] 2012-01-09 14:08:47,226 ColumnFamilyStore.java (line 692) Enqueuing flush of Memtable-URLs.URLs_1_idx@1347180703(16324791/156973287 serialized/live bytes, 173288 ops)",
      "INFO [FlushWriter:23] 2012-01-09 14:08:47,236 Memtable.java (line 240) Writing Memtable-URLs.URLs_1_idx@1347180703(16324791/156973287 serialized/live bytes, 173288 ops)",
      "INFO [pool-1-thread-1] 2012-01-09 14:08:51,003 Memtable.java (line 180) CFS(Keyspace='kev3', ColumnFamily='URLs.URLs_1_idx') liveRatio is 7.692510757866615 (just-counted was 4.512127842861816). calculation took 8648ms for 97329 columns",
      "INFO [FlushWriter:23] 2012-01-09 14:08:54,360 Memtable.java (line 277) Completed flushing /ke/cassandra/data/kev3/URLs.URLs_1_idx-hc-143-Data.db (26375495 bytes)",
      "INFO [ScheduledTasks:1] 2012-01-09 14:08:55,566 GCInspector.java (line 123) GC for ParNew: 206 ms for 1 collections, 934108624 used; max is 2034237440",
      "INFO [OptionalTasks:1] 2012-01-09 14:08:57,289 MeteredFlusher.java (line 62) flushing high-traffic column family CFS(Keyspace='kev3', ColumnFamily='URLs') (estimated 188842513 bytes)",
      "INFO [OptionalTasks:1] 2012-01-09 14:08:57,297 ColumnFamilyStore.java (line 692) Enqueuing flush of Memtable-URLs.URLs_1_idx@164871630(19662738/189069779 serialized/live bytes, 208494 ops)",
      "INFO [FlushWriter:23] 2012-01-09 14:08:57,297 Memtable.java (line 240) Writing Memtable-URLs.URLs_1_idx@164871630(19662738/189069779 serialized/live bytes, 208494 ops)",
      "INFO [ScheduledTasks:1] 2012-01-09 14:08:57,619 GCInspector.java (line 123) GC for ParNew: 402 ms for 2 collections, 981893424 used; max is 2034237440",
      "INFO [FlushWriter:23] 2012-01-09 14:09:05,944 Memtable.java (line 277) Completed flushing /ke/cassandra/data/kev3/URLs.URLs_1_idx-hc-144-Data.db (31755390 bytes)",
      "INFO [OptionalTasks:1] 2012-01-09 14:09:06,447 MeteredFlusher.java (line 62) flushing high-traffic column family CFS(Keyspace='kev3', ColumnFamily='URLs') (estimated 174605041 bytes)",
      "INFO [OptionalTasks:1] 2012-01-09 14:09:06,447 ColumnFamilyStore.java (line 692) Enqueuing flush of Memtable-URLs.URLs_1_idx@284469330(18158445/174605041 serialized/live bytes, 192702 ops)",
      "INFO [FlushWriter:23] 2012-01-09 14:09:06,447 Memtable.java (line 240) Writing Memtable-URLs.URLs_1_idx@284469330(18158445/174605041 serialized/live bytes, 192702 ops)",
      "ERROR [CompactionExecutor:260] 2012-01-09 14:09:06,448 AbstractCassandraDaemon.java (line 133) Fatal exception in thread Thread[CompactionExecutor:260,1,RMI Runtime]"
    ],
    "stack_traces": [
      "java.util.concurrent.ExecutionException: java.lang.AssertionError\nat java.util.concurrent.FutureTask$Sync.innerGet(FutureTask.java:222)\nat java.util.concurrent.FutureTask.get(FutureTask.java:83)\nat org.apache.cassandra.db.compaction.CompactionManager.performAllSSTableOperation(CompactionManager.java:203)\nat org.apache.cassandra.db.compaction.CompactionManager.performCleanup(CompactionManager.java:237)\nat org.apache.cassandra.db.ColumnFamilyStore.forceCleanup(ColumnFamilyStore.java:958)\nat org.apache.cassandra.service.StorageService.forceTableCleanup(StorageService.java:1604)\nat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)\nat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\nat java.lang.reflect.Method.invoke(Method.java:597)\nat com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:93)\nat com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:27)\nat com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:208)\nat com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:120)\nat com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:262)\nat com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:836)\nat com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:761)\nat javax.management.remote.rmi.RMIConnectionImpl.doOperation(RMIConnectionImpl.java:1427)\nat javax.management.remote.rmi.RMIConnectionImpl.access$200(RMIConnectionImpl.java:72)\nat javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(RMIConnectionImpl.java:1265)\nat javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(RMIConnectionImpl.java:1360)\nat javax.management.remote.rmi.RMIConnectionImpl.invoke(RMIConnectionImpl.java:788)\nat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)\nat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\nat java.lang.reflect.Method.invoke(Method.java:597)\nat sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:305)\nat sun.rmi.transport.Transport$1.run(Transport.java:159)\nat java.security.AccessController.doPrivileged(Native Method)\nat sun.rmi.transport.Transport.serviceCall(Transport.java:155)\nat sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:535)\nat sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:790)\nat sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:649)\nat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\nat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\nat java.lang.Thread.run(Thread.java:662)\nCaused by: java.lang.AssertionError\nat org.apache.cassandra.db.Memtable.put(Memtable.java:136)\nat org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:780)\nat org.apache.cassandra.db.index.keys.KeysIndex.deleteColumn(KeysIndex.java:82)\nat org.apache.cassandra.db.index.SecondaryIndexManager.deleteFromIndexes(SecondaryIndexManager.java:438)\nat org.apache.cassandra.db.compaction.CompactionManager.doCleanupCompaction(CompactionManager.java:754)\nat org.apache.cassandra.db.compaction.CompactionManager.access$300(CompactionManager.java:63)\nat org.apache.cassandra.db.compaction.CompactionManager$5.perform(CompactionManager.java:241)\nat org.apache.cassandra.db.compaction.CompactionManager$2.call(CompactionManager.java:182)\nat java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)\nat java.util.concurrent.FutureTask.run(FutureTask.java:138)\n... 3 more",
      "java.lang.AssertionError\nat org.apache.cassandra.db.Memtable.put(Memtable.java:136)\nat org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:780)\nat org.apache.cassandra.db.index.keys.KeysIndex.deleteColumn(KeysIndex.java:82)\nat org.apache.cassandra.db.index.SecondaryIndexManager.deleteFromIndexes(SecondaryIndexManager.java:438)\nat org.apache.cassandra.db.compaction.CompactionManager.doCleanupCompaction(CompactionManager.java:754)\nat org.apache.cassandra.db.compaction.CompactionManager.access$300(CompactionManager.java:63)\nat org.apache.cassandra.db.compaction.CompactionManager$5.perform(CompactionManager.java:241)\nat org.apache.cassandra.db.compaction.CompactionManager$2.call(CompactionManager.java:182)\nat java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)\nat java.util.concurrent.FutureTask.run(FutureTask.java:138)\nat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\nat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\nat java.lang.Thread.run(Thread.java:662)"
    ]
  },
  {
    "file": "Cassandra-4223.docx",
    "title": "Cassandra-4223",
    "version": "cassandra-1.0.10",
    "Description": "Non Unique Streaming session ID's",
    "logs": [
      "INFO [RMI TCP Connection(14)-50.28.20.137] 2012-05-04 01:40:33,878 StorageService.java (line 668) MOVING: fetching new ranges and streaming old ranges",
      "DEBUG [RMI TCP Connection(14)-50.28.20.137] 2012-05-04 01:40:33,878 StreamIn.java (line 68) Requesting from /192.168.1.7 ranges (24504928094253812322316810666345578171,28356863910078205288614550619314017619]",
      "DEBUG [Thread-28] 2012-05-04 01:40:33,928 StreamInSession.java (line 97) Adding file /var/lib/cassandra/data/FMM_Studio/PartsData-hc-1-Data.db to Stream Request queue",
      "DEBUG [Thread-28] 2012-05-04 01:40:33,938 IncomingStreamReader.java (line 77) Receiving stream",
      "DEBUG [Thread-28] 2012-05-04 01:40:33,938 IncomingStreamReader.java (line 78) Creating file for /var/lib/cassandra/data/FMM_Studio/PartsData-tmp-hc-154-Data.db with 1 estimated keys",
      "DEBUG [Thread-28] 2012-05-04 01:40:33,938 IncomingStreamReader.java (line 85) Estimated keys 1",
      "DEBUG [Thread-28] 2012-05-04 01:40:33,998 StreamInSession.java (line 105) Finished /var/lib/cassandra/data/FMM_Studio/PartsData-hc-1-Data.db sections=1 progress=305641/305641 - 100%. Sending ack to org.apache.cassandra.streaming.StreamInSession@3ea5c544",
      "INFO [Thread-28] 2012-05-04 01:40:33,998 StreamInSession.java (line 162) Finished streaming session 26132848816442266 from /192.168.1.7",
      "DEBUG [Thread-29] 2012-05-04 01:40:36,468 StreamInSession.java (line 97) Adding file /var/lib/cassandra/data/OpsCenter/rollups60-hc-6-Data.db to Stream Request queue",
      "DEBUG [Thread-29] 2012-05-04 01:40:36,468 IncomingStreamReader.java (line 77) Receiving stream",
      "DEBUG [StreamStage:1] 2012-05-03 21:40:33,897 StreamRequestVerbHandler.java (line 43) Received a StreamRequestMessage from /192.168.1.9",
      "INFO [StreamStage:1] 2012-05-03 21:40:33,897 StreamOut.java (line 114) Beginning transfer to /192.168.1.9",
      "DEBUG [StreamStage:1] 2012-05-03 21:40:33,897 StreamOut.java (line 115) Ranges are (24504928094253812322316810666345578171,28356863910078205288614550619314017619]",
      "DEBUG [StreamStage:1] 2012-05-03 21:40:33,927 StreamRequestVerbHandler.java (line 43) Received a StreamRequestMessage from /192.168.1.9",
      "INFO [StreamStage:1] 2012-05-03 21:40:33,927 StreamOut.java (line 114) Beginning transfer to /192.168.1.9",
      "DEBUG [StreamStage:1] 2012-05-03 21:40:33,927 StreamOut.java (line 115) Ranges are (24504928094253812322316810666345578171,28356863910078205288614550619314017619]",
      "DEBUG [Streaming:1] 2012-05-03 21:40:33,997 FileStreamTask.java (line 106) Done streaming /var/lib/cassandra/data/FMM_Studio/PartsData-hc-1-Data.db sections=1 progress=305641/305641 - 100%",
      "DEBUG [MiscStage:1] 2012-05-03 21:40:33,997 StreamReplyVerbHandler.java (line 47) Received StreamReply StreamReply(sessionId=26132848816442266, file='/var/lib/cassandra/data/FMM_Studio/PartsData-hc-1-Data.db', action=FILE_FINISHED)",
      "ERROR [MiscStage:1] 2012-05-03 21:40:34,027 AbstractCassandraDaemon.java (line 139) Fatal exception in thread Thread[MiscStage:1,5,main]",
      "DEBUG [MiscStage:2] 2012-05-03 21:40:34,047 StreamReplyVerbHandler.java (line 47) Received StreamReply StreamReply(sessionId=26132848816442266, file='', action=SESSION_FINISHED)",
      "INFO [FlushWriter:33] 2012-05-03 21:40:35,717 Memtable.java (line 246) Writing Memtable-pdps@1010287800(7600673/192386927 serialized/live bytes, 245183 ops)",
      "INFO [FlushWriter:33] 2012-05-03 21:40:36,387 Memtable.java (line 283) Completed flushing /var/lib/cassandra/data/OpsCenter/pdps-hc-7-Data.db (7880098 bytes)",
      "INFO [StreamStage:1] 2012-05-03 21:40:36,467 StreamOut.java (line 114) Beginning transfer to /192.168.1.9",
      "DEBUG [Streaming:1] 2012-05-03 21:40:36,467 FileStreamTask.java (line 106) Done streaming /var/lib/cassandra/data/OpsCenter/rollups7200-hc-3-Data.db sections=1 progress=69548/69548 - 100%",
      "DEBUG [MiscStage:2] 2012-05-03 21:40:36,497 StreamReplyVerbHandler.java (line 47) Received StreamReply StreamReply(sessionId=26132848816442266, file='/var/lib/cassandra/data/OpsCenter/rollups7200-hc-3-Data.db', action=FILE_FINISHED)",
      "ERROR [MiscStage:2] 2012-05-03 21:40:36,497 AbstractCassandraDaemon.java (line 139) Fatal exception in thread Thread[MiscStage:2,5,main]"
    ],
    "stack_traces": [
      "java.lang.IllegalStateException: target reports current file is /var/lib/cassandra/data/FMM_Studio/PartsData-hc-1-Data.db but is null\n\t\tat org.apache.cassandra.streaming.StreamOutSession.validateCurrentFile(StreamOutSession.java:195)\n\t\tat org.apache.cassandra.streaming.StreamReplyVerbHandler.doVerb(StreamReplyVerbHandler.java:58)\n\t\tat org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:59)\n\t\tat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(Unknown Source)\n\t\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)\n\t\tat java.lang.Thread.run(Unknown Source)",
      "java.lang.IllegalStateException: target reports current file is /var/lib/cassandra/data/OpsCenter/rollups7200-hc-3-Data.db but is null\n\t\tat org.apache.cassandra.streaming.StreamOutSession.validateCurrentFile(StreamOutSession.java:195)\n\t\tat org.apache.cassandra.streaming.StreamReplyVerbHandler.doVerb(StreamReplyVerbHandler.java:58)\n\t\tat org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:59)\n\t\tat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(Unknown Source)\n\t\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)\n\t\tat java.lang.Thread.run(Unknown Source)"
    ]
  },
  {
    "file": "Cassandra-4675.docx",
    "title": "Cassandra-4675",
    "version": "cassandra-1.1.5",
    "Description": "NPE in NTS when using LQ against a node (DC) that doesn't have replica",
    "logs": [
      "ERROR [Thrift:3] 2012-09-17 18:15:16,868 Cassandra.java (line 2999) Internal error processing get"
    ],
    "stack_traces": [
      "org.apache.thrift.TApplicationException: Internal error processing get\nat org.apache.thrift.TApplicationException.read(TApplicationException.java:108)\nat org.apache.cassandra.thrift.Cassandra$Client.recv_get(Cassandra.java:511)\nat org.apache.cassandra.thrift.Cassandra$Client.get(Cassandra.java:492)\nat org.apache.cassandra.cli.CliClient.executeGet(CliClient.java:648)\nat org.apache.cassandra.cli.CliClient.executeCLIStatement(CliClient.java:209)\nat org.apache.cassandra.cli.CliMain.processStatementInteractive(CliMain.java:220)\nat org.apache.cassandra.cli.CliMain.main(CliMain.java:348)",
      "java.lang.NullPointerException\nat org.apache.cassandra.locator.NetworkTopologyStrategy.getReplicationFactor(NetworkTopologyStrategy.java:142)\nat org.apache.cassandra.service.DatacenterReadCallback.determineBlockFor(DatacenterReadCallback.java:90)\nat org.apache.cassandra.service.ReadCallback.<init>(ReadCallback.java:67)\nat org.apache.cassandra.service.DatacenterReadCallback.<init>(DatacenterReadCallback.java:63)\nat org.apache.cassandra.service.StorageProxy.getReadCallback(StorageProxy.java:775)\nat org.apache.cassandra.service.StorageProxy.fetchRows(StorageProxy.java:609)\nat org.apache.cassandra.service.StorageProxy.read(StorageProxy.java:564)\nat org.apache.cassandra.thrift.CassandraServer.readColumnFamily(CassandraServer.java:128)\nat org.apache.cassandra.thrift.CassandraServer.internal_get(CassandraServer.java:383)\nat org.apache.cassandra.thrift.CassandraServer.get(CassandraServer.java:401)\nat org.apache.cassandra.thrift.Cassandra$Processor$get.process(Cassandra.java:2989)\nat org.apache.cassandra.thrift.Cassandra$Processor.process(Cassandra.java:2889)\nat org.apache.cassandra.thrift.CustomTThreadPoolServer$WorkerProcess.run(CustomTThreadPoolServer.java:187)\nat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\nat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)"
    ]
  },
  {
    "file": "Cassandra-4774.docx",
    "title": "Cassandra-4774",
    "version": "cassandra-1.2.8",
    "Description": "IndexOutOfBoundsException in org.apache.cassandra.gms.Gossiper.sendGossip",
    "logs": [
      "ERROR [GossipTasks:1] 2012-10-06 10:47:48,390 Gossiper.java (line 169) Gossip error",
      "WARN [ScheduledTasks:1] 2013-07-07 19:37:03,834 GCInspector.java (line 145) Heap is 0.9552299542288667 full.",
      "WARN [ScheduledTasks:1] 2013-07-07 19:37:03,834 StorageService.java (line 2855) Flushing CFS(Keyspace='x', ColumnFamily='x') to relieve memory pressure",
      "INFO [ScheduledTasks:1] 2013-07-07 19:37:03,834 ColumnFamilyStore.java (line 659) Enqueuing flush of Memtable-x@766608353(261434/1801824 serialized/live bytes, 5150 ops)",
      "INFO [GossipStage:1] 2013-07-07 19:37:05,125 Gossiper.java (line 816) InetAddress /10.x.x.x is now UP",
      "INFO [GossipStage:1] 2013-07-07 19:37:05,146 Gossiper.java (line 816) InetAddress /10.x.x.x is now UP",
      "ERROR [GossipTasks:1] 2013-07-07 19:37:05,155 Gossiper.java (line 171) Gossip error"
    ],
    "stack_traces": [
      "java.lang.IndexOutOfBoundsException: Index: 13, Size: 5\nat java.util.ArrayList.RangeCheck(ArrayList.java:547)\nat java.util.ArrayList.get(ArrayList.java:322)\nat org.apache.cassandra.gms.Gossiper.sendGossip(Gossiper.java:541)\nat org.apache.cassandra.gms.Gossiper.doGossipToUnreachableMember(Gossiper.java:575)\nat org.apache.cassandra.gms.Gossiper.access$300(Gossiper.java:59)\nat org.apache.cassandra.gms.Gossiper$GossipTask.run(Gossiper.java:141)\nat org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor$UncomplainingRunnable.run(DebuggableScheduledThreadPoolExecutor.java:79)\nat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)\nat java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:317)\nat java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)\nat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)\nat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:180)\nat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:204)\nat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\nat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\nat java.lang.Thread.run(Thread.java:662)",
      "java.lang.IndexOutOfBoundsException: Index: 10, Size: 10\n\tat java.util.ArrayList.RangeCheck(ArrayList.java:547)\n\tat java.util.ArrayList.get(ArrayList.java:322)\n\tat org.apache.cassandra.gms.Gossiper.sendGossip(Gossiper.java:560)\n\tat org.apache.cassandra.gms.Gossiper.doGossipToUnreachableMember(Gossiper.java:594)\n\tat org.apache.cassandra.gms.Gossiper.access$300(Gossiper.java:61)\n\tat org.apache.cassandra.gms.Gossiper$GossipTask.run(Gossiper.java:143)\n\tat org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor$UncomplainingRunnable.run(DebuggableScheduledThreadPoolExecutor.java:79)\n\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)\n\tat java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:317)\n\tat java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)\n\tat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)\n\tat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:180)\n\tat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:204)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\n\tat java.lang.Thread.run(Thread.java:662)"
    ]
  },
  {
    "file": "Cassandra-5105.docx",
    "title": "Cassandra-5105",
    "version": "cassandra-1.2.1",
    "Description": "repair -pr throws EOFException",
    "logs": [
      "INFO 15:53:36,532 [streaming task #aa70f8e0-656c-11e2-b226-d966287ae7ca] Received task from /10.80.90.51 to stream 7871 ranges to /10.80.90.53",
      "INFO 15:53:36,533 [streaming task #aa70f8e0-656c-11e2-b226-d966287ae7ca] Performing streaming repair of 7871 ranges with /10.80.90.53",
      "INFO 15:53:43,216 Stream context metadata [/var/lib/cassandra/data/MyBusinessKeyspace/MyBusinessHistoryCF/MyBusinessKeyspace-MyBusinessHistoryCF-ia-4840-Data.db sections=5086 progress=0/350803667 - 0%, /var/lib/cassandra/data/MyBusinessKeyspace/MyBusinessHistoryCF/MyBusinessKeyspace-MyBusinessHistoryCF-ia-4787-Data.db sections=6079 progress=0/1160848303 - 0%, /var/lib/cassandra/data/MyBusinessKeyspace/MyBusinessHistoryCF/MyBusinessKeyspace-MyBusinessHistoryCF-ia-4825-Data.db sections=5526 progress=0/571701570 - 0%, /var/lib/cassandra/data/MyBusinessKeyspace/MyBusinessHistoryCF/MyBusinessKeyspace-MyBusinessHistoryCF-ia-4879-Data.db sections=3043 progress=0/3086625 - 0%], 11 sstables.",
      "INFO 15:53:43,217 Streaming to /10.80.90.53",
      "INFO 15:53:43,325 Beginning transfer to /10.80.90.51",
      "INFO 15:53:43,362 Flushing memtables for [CFS(Keyspace='MyBusinessKeyspace', ColumnFamily='MyBusinessHistoryCF')]...",
      "INFO 15:53:43,363 Enqueuing flush of Memtable-MyBusinessHistoryCF@554695962(2424273/24900316 serialized/live bytes, 51289 ops)",
      "INFO 15:53:43,366 Writing Memtable-MyBusinessHistoryCF@554695962(2424273/24900316 serialized/live bytes, 51289 ops)",
      "INFO 15:53:43,558 Completed flushing /var/lib/cassandra/data/MyBusinessKeyspace/MyBusinessHistoryCF/MyBusinessKeyspace-MyBusinessHistoryCF-ia-4881-Data.db (2083586 bytes) for commitlog position ReplayPosition(segmentId=1358949372387, position=2780498)",
      "INFO 15:53:51,056 Stream context metadata [/var/lib/cassandra/data/MyBusinessKeyspace/MyBusinessHistoryCF/MyBusinessKeyspace-MyBusinessHistoryCF-ia-4787-Data.db sections=6645 progress=0/1303785158 - 0%, /var/lib/cassandra/data/MyBusinessKeyspace/MyBusinessHistoryCF/MyBusinessKeyspace-MyBusinessHistoryCF-ia-4840-Data.db sections=5775 progress=0/392625457 - 0%, /var/lib/cassandra/data/MyBusinessKeyspace/MyBusinessHistoryCF/MyBusinessKeyspace-MyBusinessHistoryCF-ia-4879-Data.db sections=4083 progress=0/3265613 - 0%], 12 sstables.",
      "INFO 15:53:51,057 Streaming to /10.80.90.51",
      "INFO 15:54:19,013 Successfully sent /var/lib/cassandra/data/MyBusinessKeyspace/MyBusinessHistoryCF/MyBusinessKeyspace-MyBusinessHistoryCF-ia-4840-Data.db to /10.80.90.53",
      "ERROR 15:54:46,686 Exception in thread Thread[Thread-3087,5,main]",
      "INFO 15:55:37,245 CFS(Keyspace='OpsCenter', ColumnFamily='pdps') liveRatio is 2.3633228082745292 (just-counted was 2.3633228082745292). calculation took 2ms for 1256 columns",
      "INFO 15:56:30,149 Successfully sent /var/lib/cassandra/data/MyBusinessKeyspace/MyBusinessHistoryCF/MyBusinessKeyspace-MyBusinessHistoryCF-ia-4787-Data.db to /10.80.90.53",
      "INFO 15:56:32,401 Successfully sent /var/lib/cassandra/data/MyBusinessKeyspace/MyBusinessHistoryCF/MyBusinessKeyspace-MyBusinessHistoryCF-ia-4878-Data.db to /10.80.90.53",
      "INFO 15:53:43,296 Enqueuing flush of Memtable-MyBusinessHistoryCF@709374075(7410300/48461437 serialized/live bytes, 155356 ops) [cassandra01(/10.80.90.51)]",
      "INFO 15:53:43,304 Writing Memtable-MyBusinessHistoryCF@709374075(7410300/48461437 serialized/live bytes, 155356 ops) [cassandra01(/10.80.90.51)]",
      "INFO 15:53:43,688 Completed flushing /var/lib/cassandra/data/MyBusinessKeyspace/MyBusinessHistoryCF/MyBusinessKeyspace-MyBusinessHistoryCF-ia-4952-Data.db (4474554 bytes) for commitlog position ReplayPosition(segmentId=1358949134910, position=13864741) [cassandra01(/10.80.90.51)]",
      "INFO 15:54:09,160 Successfully sent /var/lib/cassandra/data/MyBusinessKeyspace/MyBusinessHistoryCF/MyBusinessKeyspace-MyBusinessHistoryCF-ia-4922-Data.db to /10.80.90.52 [cassandra01(/10.80.90.51)]",
      "INFO 15:54:11,605 Successfully sent /var/lib/cassandra/data/MyBusinessKeyspace/MyBusinessHistoryCF/MyBusinessKeyspace-MyBusinessHistoryCF-ia-4922-Data.db to /10.80.90.53 [cassandra01(/10.80.90.51)]",
      "ERROR 15:54:46,682 Exception in thread Thread[Streaming to /10.80.90.52:2,5,main] [cassandra01(/10.80.90.51)]"
    ],
    "stack_traces": [
      "java.lang.RuntimeException: Last written key DecoratedKey(153906576608468125601485890282698016632, 72736b3a67726f75703a73656375726974793a496e74657276656e74696f6e3a496e7472616461793a66306135386333352d353262312d343361642d396430332d6130636630306330306565633a3937383a313330313137) >= current key DecoratedKey(33745288399064288388334698406389712581, bec0000000220a0b08f494e9b2b582a3301005121308......0b08f4bab8c7ddd8a330100500080000013c6679e1e10200093a805108d6a10004d3f04c1c040d0000000d0a0b08bedfbbd0000000450fe4ef100080000013c3d58dcf6010004d34fa32017330000000450fe4f3300080000013c3d5921a1010004d34fa42bd8c00000000450fe4f4500080000013c3d59abe1010004d34fa64b77240000000450fe4f6900080000013c3d5a29cd010004d34fa834f5ee0000000450fe4f8900080000013c3d5b24) writing into /var/lib/cassandra/data/MyBusinessKeyspace/MyBusinessHistoryCF/MyBusinessKeyspace-MyBusinessHistoryCF-tmp-ia-4882-Data.db\n\t\t\t\t\tat org.apache.cassandra.io.sstable.SSTableWriter.beforeAppend(SSTableWriter.java:133)\n\t\t\t\t\tat org.apache.cassandra.io.sstable.SSTableWriter.appendFromStream(SSTableWriter.java:209)\n\t\t\t\t\tat org.apache.cassandra.streaming.IncomingStreamReader.streamIn(IncomingStreamReader.java:179)\n\t\t\t\t\tat org.apache.cassandra.streaming.IncomingStreamReader.read(IncomingStreamReader.java:122)\n\t\t\t\t\tat org.apache.cassandra.net.IncomingTcpConnection.stream(IncomingTcpConnection.java:226)\n\t\t\t\t\tat org.apache.cassandra.net.IncomingTcpConnection.handleStream(IncomingTcpConnection.java:166)\n\t\t\t\t\tat org.apache.cassandra.net.IncomingTcpConnection.run(IncomingTcpConnection.java:66)",
      "java.lang.RuntimeException: java.io.IOException: Broken pipe\n\t\t\t\t\tat com.google.common.base.Throwables.propagate(Throwables.java:160)\n\t\t\t\t\tat org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:32)\n\t\t\t\t\tat java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)\n\t\t\t\t\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)\n\t\t\t\t\tat java.lang.Thread.run(Unknown Source)\nCaused by: java.io.IOException: Broken pipe\n\t\t\t\t\tat sun.nio.ch.FileChannelImpl.transferTo0(Native Method)\n\t\t\t\t\tat sun.nio.ch.FileChannelImpl.transferToDirectly(Unknown Source)\n\t\t\t\t\tat sun.nio.ch.FileChannelImpl.transferTo(Unknown Source)\n\t\t\t\t\tat org.apache.cassandra.streaming.compress.CompressedFileStreamTask.stream(CompressedFileStreamTask.java:90)\n\t\t\t\t\tat org.apache.cassandra.streaming.FileStreamTask.runMayThrow(FileStreamTask.java:91)\n\t\t\t\t\tat org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)\n\t\t\t\t\t... 3 more"
    ]
  },
  {
    "file": "Cassandra-5133.docx",
    "title": "Cassandra-5133",
    "version": "cassandra-1.2.0",
    "Description": "Nodes can't rejoin after stopping, when using GossipingPropertyFileSnitch",
    "logs": [
      "ERROR 05:45:39,305 Exception encountered during startup",
      "ERROR 02:58:45,376 Exception in thread Thread[WRITE-cassandra-1/10.179.65.102,5,main]"
    ],
    "stack_traces": [
      "java.lang.RuntimeException: Could not retrieve DC for /10.114.18.51 from gossip and PFS compatibility is disabled\nat org.apache.cassandra.locator.GossipingPropertyFileSnitch.getDatacenter(GossipingPropertyFileSnitch.java:109)\nat org.apache.cassandra.locator.DynamicEndpointSnitch.getDatacenter(DynamicEndpointSnitch.java:127)\nat org.apache.cassandra.locator.TokenMetadata$Topology.addEndpoint(TokenMetadata.java:1040)\nat org.apache.cassandra.locator.TokenMetadata.updateNormalTokens(TokenMetadata.java:185)\nat org.apache.cassandra.locator.TokenMetadata.updateNormalTokens(TokenMetadata.java:157)\nat org.apache.cassandra.service.StorageService.initServer(StorageService.java:441)\nat org.apache.cassandra.service.StorageService.initServer(StorageService.java:397)\nat org.apache.cassandra.service.CassandraDaemon.setup(CassandraDaemon.java:309)\nat org.apache.cassandra.service.CassandraDaemon.activate(CassandraDaemon.java:397)\nat org.apache.cassandra.service.CassandraDaemon.main(CassandraDaemon.java:440)",
      "java.lang.RuntimeException: Could not retrieve DC for cassandra-1/10.179.65.102 from gossip and PFS compatibility is disabled\n\tat org.apache.cassandra.locator.GossipingPropertyFileSnitch.getDatacenter(GossipingPropertyFileSnitch.java:80)\n\tat org.apache.cassandra.locator.DynamicEndpointSnitch.getDatacenter(DynamicEndpointSnitch.java:127)\n\tat org.apache.cassandra.net.OutboundTcpConnection.isLocalDC(OutboundTcpConnection.java:73)\n\tat org.apache.cassandra.net.OutboundTcpConnection.connect(OutboundTcpConnection.java:266)\n\tat org.apache.cassandra.net.OutboundTcpConnection.run(OutboundTcpConnection.java:138)"
    ]
  },
  {
    "file": "Cassandra-5229.docx",
    "title": "Cassandra-5229",
    "version": "cassandra-1.2.4",
    "Description": "After an IOException is thrown during streaming, streaming tasks hang in netstats",
    "logs": [
      "INFO [AntiEntropyStage:1] 2013-02-07 11:23:44,717 StreamOut.java (line 151) Stream context metadata [/data/cassandra/evidence/fingerprints/evidence-fingerprints-ib-5-Data.db sections=3068 progress=0/2785204713 - 0%, /data2/cassandra/evidence/fingerprints/evidence-fingerprints-ib-25-Data.db sections=2696 progress=0/758409465 - 0%, /data/cassandra/evidence/fingerprints/evidence-fingerprints-ib-60-Data.db sections=3099 progress=0/238876436 - 0%, /data2/cassandra/evidence/fingerprints/evidence-fingerprints-ib-63-Data.db sections=1166 progress=0/2125323 - 0%, /data2/cassandra/evidence/fingerprints/evidence-fingerprints-ib-38-Data.db sections=2507 progress=0/515992757 - 0%, /data2/cassandra/evidence/fingerprints/evidence-fingerprints-ib-26-Data.db sections=3153 progress=0/994857654 - 0%, /data2/cassandra/evidence/fingerprints/evidence-fingerprints-ib-57-Data.db sections=3116 progress=0/129398170 - 0%, /data2/cassandra/evidence/fingerprints/evidence-fingerprints-ib-58-Data.db sections=217 progress=0/72286 - 0%, /data/cassandra/evidence/fingerprints/evidence-fingerprints-ib-59-Data.db sections=3146 progress=0/3357709019 - 0%], 27 sstables.",
      "INFO [AntiEntropyStage:1] 2013-02-07 11:23:52,964 StreamOut.java (line 151) Stream context metadata [/data/cassandra/evidence/fingerprints/evidence-fingerprints-ib-5-Data.db sections=2930 progress=0/2799914560 - 0%, /data2/cassandra/evidence/fingerprints/evidence-fingerprints-ib-25-Data.db sections=2590 progress=0/761266059 - 0%, /data/cassandra/evidence/fingerprints/evidence-fingerprints-ib-60-Data.db sections=2956 progress=0/241362497 - 0%, /data2/cassandra/evidence/fingerprints/evidence-fingerprints-ib-63-Data.db sections=1153 progress=0/2125323 - 0%, /data2/cassandra/evidence/fingerprints/evidence-fingerprints-ib-38-Data.db sections=2422 progress=0/522126371 - 0%, /data2/cassandra/evidence/fingerprints/evidence-fingerprints-ib-26-Data.db sections=3004 progress=0/998401202 - 0%, /data2/cassandra/evidence/fingerprints/evidence-fingerprints-ib-57-Data.db sections=2974 progress=0/129722346 - 0%, /data2/cassandra/evidence/fingerprints/evidence-fingerprints-ib-58-Data.db sections=220 progress=0/72286 - 0%, /data/cassandra/evidence/fingerprints/evidence-fingerprints-ib-59-Data.db sections=2998 progress=0/3375554099 - 0%], 27 sstables.",
      "INFO [MiscStage:1] 2013-02-07 11:23:38,022 StreamOut.java (line 151) Stream context metadata [/data/cassandra/evidence/fingerprints/evidence-fingerprints-ia-472-Data.db sections=1727 progress=0/210208515 - 0%, /var/lib/cassandra/data2/evidence/fingerprints/evidence-fingerprints-ib-919-Data.db sections=1746 progress=0/119438030 - 0%, /data/cassandra/evidence/fingerprints/evidence-fingerprints-ib-920-Data.db sections=1681 progress=0/54498226 - 0%, /var/lib/cassandra/data2/evidence/fingerprints/evidence-fingerprints-ib-922-Data.db sections=16 progress=0/13490 - 0%, /var/lib/cassandra/data2/evidence/fingerprints/evidence-fingerprints-ib-918-Data.db sections=632 progress=0/70019542 - 0%, /var/lib/cassandra/data2/evidence/fingerprints/evidence-fingerprints-ib-921-Data.db sections=1644 progress=0/39870238 - 0%, /data/cassandra/evidence/fingerprints/evidence-fingerprints-ib-497-Data.db sections=1569 progress=0/208331077 - 0%, /var/lib/cassandra/data2/evidence/fingerprints/evidence-fingerprints-ib-923-Data.db sections=1572 progress=0/30870478 - 0%, /var/lib/cassandra/data2/evidence/fingerprints/evidence-fingerprints-ib-925-Data.db sections=167 progress=0/1845123 - 0%, /var/lib/cassandra/data2/evidence/fingerprints/evidence-fingerprints-ib-703-Data.db sections=1574 progress=0/287386471 - 0%, /var/lib/cassandra/data2/evidence/fingerprints/evidence-fingerprints-ib-913-Data.db sections=811 progress=0/103776521 - 0%, /var/lib/cassandra/data2/evidence/fingerprints/evidence-fingerprints-ib-915-Data.db sections=1539 progress=0/141864261 - 0%], 12 sstables.",
      "INFO [MiscStage:1] 2013-02-07 11:23:49,938 StreamOut.java (line 151) Stream context metadata [/var/lib/cassandra/data2/evidence/fingerprints/evidence-fingerprints-ib-919-Data.db sections=3153 progress=0/994857654 - 0%, /var/lib/cassandra/data2/evidence/fingerprints/evidence-fingerprints-ib-918-Data.db sections=2507 progress=0/515992757 - 0%, /var/lib/cassandra/data2/evidence/fingerprints/evidence-fingerprints-ib-923-Data.db sections=3153 progress=0/131969347 - 0%, /var/lib/cassandra/data2/evidence/fingerprints/evidence-fingerprints-ib-703-Data.db sections=3068 progress=0/2784807967 - 0%, /var/lib/cassandra/data2/evidence/fingerprints/evidence-fingerprints-ib-913-Data.db sections=2696 progress=0/758409465 - 0%, /data/cassandra/evidence/fingerprints/evidence-fingerprints-ia-472-Data.db sections=3150 progress=0/1792868996 - 0%, /data/cassandra/evidence/fingerprints/evidence-fingerprints-ib-920-Data.db sections=3123 progress=0/240094510 - 0%, /var/lib/cassandra/data2/evidence/fingerprints/evidence-fingerprints-ib-929-Data.db sections=18 progress=0/29468 - 0%, /var/lib/cassandra/data2/evidence/fingerprints/evidence-fingerprints-ib-922-Data.db sections=217 progress=0/13490 - 0%, /var/lib/cassandra/data2/evidence/fingerprints/evidence-fingerprints-ib-921-Data.db sections=3124 progress=0/130320291 - 0%, /data/cassandra/evidence/fingerprints/evidence-fingerprints-ib-497-Data.db sections=3055 progress=0/1749539598 - 0%, /var/lib/cassandra/data2/evidence/fingerprints/evidence-fingerprints-ib-925-Data.db sections=1608 progress=0/13357410 - 0%, /var/lib/cassandra/data2/evidence/fingerprints/evidence-fingerprints-ib-915-Data.db sections=3096 progress=0/1241397203 - 0%], 13 sstables.",
      "INFO [Streaming to /10.8.25.132:2] 2013-02-07 11:24:18,780 StreamReplyVerbHandler.java (line 44) Successfully sent /var/lib/cassandra/data2/evidence/fingerprints/evidence-fingerprints-ib-919-Data.db to /10.8.25.132",
      "INFO 2013-02-07 11:24:xx,xxx Streaming from: /10.8.30.13 | evidence: /var/lib/cassandra/data2/evidence/fingerprints/evidence-fingerprints-ib-919-Data.db sections=3153 progress=218225400/994857654 - 21%",
      "INFO 2013-02-07 11:24:xx,xxx Streaming from: /10.138.12.10 | evidence: /data2/cassandra/evidence/fingerprints/evidence-fingerprints-ib-26-Data.db sections=3153 progress=218225400/994857654 - 21%",
      "ERROR [Streaming to /10.8.25.114:4] 2013-02-09 11:02:09,992 CassandraDaemon.java (line 135) Exception in thread Thread[Streaming to /10.8.25.114:4,5,main]"
    ],
    "stack_traces": [
      "java.lang.RuntimeException: java.io.IOException: Broken pipe\nat com.google.common.base.Throwables.propagate(Throwables.java:160)\nat org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:32)\nat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\nat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\nat java.lang.Thread.run(Thread.java:722)\nCaused by: java.io.IOException: Broken pipe\nat sun.nio.ch.FileChannelImpl.transferTo0(Native Method)\nat sun.nio.ch.FileChannelImpl.transferToDirectly(FileChannelImpl.java:420)\nat sun.nio.ch.FileChannelImpl.transferTo(FileChannelImpl.java:552)\nat org.apache.cassandra.streaming.compress.CompressedFileStreamTask.stream(CompressedFileStreamTask.java:90)\nat org.apache.cassandra.streaming.FileStreamTask.runMayThrow(FileStreamTask.java:91)\nat org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)\n... 3 more"
    ]
  },
  {
    "file": "Cassandra-5254.docx",
    "title": "Cassandra-5254",
    "version": "cassandra-1.1.10",
    "Description": "Nodes can be marked up after gossip sends the goodbye command",
    "logs": [
      "INFO [FlushWriter:1] 2013-02-14 10:01:10,311 Memtable.java (line 305) Completed flushing /tmp/dtest-iaYzzR/test/node1/data/system/schema_columns/system-schema_columns-hf-2-Data.db (558 bytes) for commitlog position ReplayPosition(segmentId=1360857665931, position=4770)",
      "INFO [MemoryMeter:1] 2013-02-14 10:01:10,974 Memtable.java (line 213) CFS(Keyspace='ks', ColumnFamily='cf') liveRatio is 20.488836662749705 (just-counted was 20.488836662749705). calculation took 96ms for 144 columns",
      "INFO [StorageServiceShutdownHook] 2013-02-14 10:01:11,115 Gossiper.java (line 1134) Announcing shutdown",
      "INFO [StorageServiceShutdownHook] 2013-02-14 10:01:12,118 MessagingService.java (line 549) Waiting for messaging service to quiesce",
      "INFO [GossipStage:1] 2013-02-14 10:01:12,119 Gossiper.java (line 831) InetAddress /127.0.0.3 is now dead.",
      "INFO [GossipStage:1] 2013-02-14 10:01:12,119 Gossiper.java (line 831) InetAddress /127.0.0.3 is now dead.",
      "INFO [ACCEPT-/127.0.0.3] 2013-02-14 10:01:12,119 MessagingService.java (line 705) MessagingService shutting down server thread.",
      "INFO [GossipStage:1] 2013-02-14 10:01:12,238 Gossiper.java (line 817) InetAddress /127.0.0.3 is now UP",
      "INFO [GossipTasks:1] 2013-02-14 10:01:26,386 Gossiper.java (line 831) InetAddress /127.0.0.3 is now dead."
    ],
    "stack_traces": []
  },
  {
    "file": "Cassandra-5393.docx",
    "title": "Cassandra-5393",
    "version": "cassandra-1.1.11",
    "Description": "Add retry mechanism to OTC for non-droppable_verbs",
    "logs": [
      "INFO [AntiEntropyStage:1] 2013-03-27 22:48:55,390 AntiEntropyService.java (line 239) repair #80fe25a0-9730-11e2-0000-ebe7011631ff Sending completed merkle tree to /54.246.XXX.YYY for (Geo,GeoCountryMetadata)",
      "DEBUG [WRITE-/54.246.XXX.YYY] 2013-03-27 22:48:55,392 OutboundTcpConnection.java (line 165) error writing to ec2-54-246-XXX.YYY.eu-west-1.compute.amazonaws.com/54.246.XXX.YYY"
    ],
    "stack_traces": [
      "java.net.SocketException: Connection timed out\nat java.net.SocketOutputStream.socketWrite0(Native Method)\nat java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:92)\nat java.net.SocketOutputStream.write(SocketOutputStream.java:136)\nat com.sun.net.ssl.internal.ssl.OutputRecord.writeBuffer(OutputRecord.java:358)\nat com.sun.net.ssl.internal.ssl.OutputRecord.write(OutputRecord.java:346)\nat com.sun.net.ssl.internal.ssl.SSLSocketImpl.writeRecordInternal(SSLSocketImpl.java:781)\nat com.sun.net.ssl.internal.ssl.SSLSocketImpl.writeRecord(SSLSocketImpl.java:753)\nat com.sun.net.ssl.internal.ssl.AppOutputStream.write(AppOutputStream.java:100)\nat java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:65)\nat java.io.BufferedOutputStream.write(BufferedOutputStream.java:104)\nat java.io.DataOutputStream.write(DataOutputStream.java:90)\nat java.io.FilterOutputStream.write(FilterOutputStream.java:80)\nat org.apache.cassandra.net.OutboundTcpConnection.write(OutboundTcpConnection.java:200)\nat org.apache.cassandra.net.OutboundTcpConnection.writeConnected(OutboundTcpConnection.java:152)\nat org.apache.cassandra.net.OutboundTcpConnection.run(OutboundTcpConnection.java:126)"
    ]
  },
  {
    "file": "Cassandra-5418.docx",
    "title": "Cassandra-5418",
    "version": "cassandra-1.2.4",
    "Description": "Streaming fails",
    "logs": [
      "ERROR [Streaming to /10.10.45.60:28] 2013-04-02 09:03:55,353 CassandraDaemon.java (line 132) Exception in thread Thread[Streaming to /10.10.45.60:28,5,main]",
      "ERROR [Thread-2076] 2013-04-02 09:07:12,261 CassandraDaemon.java (line 132) Exception in thread Thread[Thread-2076,5,main]",
      "ERROR [Thread-1424] 2013-04-02 09:07:12,248 CassandraDaemon.java (line 132) Exception in thread Thread[Thread-1424,5,main]",
      "ERROR [Streaming to /10.10.45.58:55] 2013-04-02 09:07:12,263 CassandraDaemon.java (line 132) Exception in thread Thread[Streaming to /10.10.45.58:55,5,main]"
    ],
    "stack_traces": [
      "java.lang.RuntimeException: java.io.EOFException\nat com.google.common.base.Throwables.propagate(Throwables.java:160)\nat org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:32)\nat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(Unknown Source)\nat java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)\nat java.lang.Thread.run(Unknown Source)\nCaused by: java.io.EOFException\nat java.io.DataInputStream.readInt(Unknown Source)\nat org.apache.cassandra.streaming.FileStreamTask.receiveReply(FileStreamTask.java:193)\nat org.apache.cassandra.streaming.compress.CompressedFileStreamTask.stream(CompressedFileStreamTask.java:114)\nat org.apache.cassandra.streaming.FileStreamTask.runMayThrow(FileStreamTask.java:91)\nat org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)\n... 3 more",
      "java.lang.AssertionError: incorrect row data size 130921 written to /var/lib/cassandra/data/EDITED/content_list/footballsite-content_list-tmp-ib-3660-Data.db; correct is 131074\nat org.apache.cassandra.io.sstable.SSTableWriter.appendFromStream(SSTableWriter.java:285)\nat org.apache.cassandra.streaming.IncomingStreamReader.streamIn(IncomingStreamReader.java:179)\nat org.apache.cassandra.streaming.IncomingStreamReader.read(IncomingStreamReader.java:122)\nat org.apache.cassandra.net.IncomingTcpConnection.stream(IncomingTcpConnection.java:238)\nat org.apache.cassandra.net.IncomingTcpConnection.handleStream(IncomingTcpConnection.java:178)\nat org.apache.cassandra.net.IncomingTcpConnection.run(IncomingTcpConnection.java:78)",
      "java.lang.AssertionError: incorrect row data size 130921 written to /var/lib/cassandra/data/EDITED/content_list/footballsite-content_list-tmp-ib-2268-Data.db; correct is 131074\nat org.apache.cassandra.io.sstable.SSTableWriter.appendFromStream(SSTableWriter.java:285)\nat org.apache.cassandra.streaming.IncomingStreamReader.streamIn(IncomingStreamReader.java:179)\nat org.apache.cassandra.streaming.IncomingStreamReader.read(IncomingStreamReader.java:122)\nat org.apache.cassandra.net.IncomingTcpConnection.stream(IncomingTcpConnection.java:238)\nat org.apache.cassandra.net.IncomingTcpConnection.handleStream(IncomingTcpConnection.java:178)\nat org.apache.cassandra.net.IncomingTcpConnection.run(IncomingTcpConnection.java:78)",
      "java.lang.RuntimeException: java.io.EOFException\nat com.google.common.base.Throwables.propagate(Throwables.java:160)\nat org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:32)\nat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(Unknown Source)\nat java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)\nat java.lang.Thread.run(Unknown Source)\nCaused by: java.io.EOFException\nat java.io.DataInputStream.readInt(Unknown Source)\nat org.apache.cassandra.streaming.FileStreamTask.receiveReply(FileStreamTask.java:193)\nat org.apache.cassandra.streaming.compress.CompressedFileStreamTask.stream(CompressedFileStreamTask.java:114)\nat org.apache.cassandra.streaming.FileStreamTask.runMayThrow(FileStreamTask.java:91)\nat org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)\n... 3 more"
    ]
  },
  {
    "file": "Cassandra-5432.docx",
    "title": "Cassandra-5432",
    "version": "cassandra-1.2.4",
    "Description": "Repair Freeze/Gossip Invisibility Issues 1.2.4",
    "logs": [
      "INFO [Thread-42214] 2013-04-05 23:30:27,785 StorageService.java (line 2379) Starting repair command #4, repairing 1 ranges for keyspace cardspring_production",
      "INFO [AntiEntropySessions:7] 2013-04-05 23:30:27,789 AntiEntropyService.java (line 652) repair #cc5a9aa0-9e48-11e2-98ba-11bde7670242 new session: will sync /X.X.X.190, /X.X.X.43, /X.X.X.56 on range (1808575600,42535295865117307932921825930779602032] for keyspace_production.[comma separated list of CFs]",
      "INFO [AntiEntropySessions:7] 2013-04-05 23:30:27,790 AntiEntropyService.java (line 858) repair #cc5a9aa0-9e48-11e2-98ba-11bde7670242 requesting merkle trees for BusinessConnectionIndicesEntries (to [/X.X.X.43, /X.X.X.56, /X.X.X.190])",
      "INFO [AntiEntropyStage:1] 2013-04-05 23:30:28,086 AntiEntropyService.java (line 214) repair #cc5a9aa0-9e48-11e2-98ba-11bde7670242 Received merkle tree for ColumnFamilyName from /X.X.X.43",
      "INFO [AntiEntropyStage:1] 2013-04-05 23:30:28,147 AntiEntropyService.java (line 214) repair #cc5a9aa0-9e48-11e2-98ba-11bde7670242 Received merkle tree for ColumnFamilyName from /X.X.X.56",
      "INFO [Thread-458] 2013-04-24 23:21:16,543 StorageService.java (line 2407) Starting repair command #1, repairing 1 ranges for keyspace app_production",
      "DEBUG [Thread-458] 2013-04-24 23:21:16,580 StorageService.java (line 2547) computing ranges for 1808575600, 7089215977519551322153637656637080005, 14178431955039102644307275311465584410, 42535295865117307932921825930779602030, 49624511842636859255075463585608106435, 56713727820156410577229101240436610840, 85070591730234615865843651859750628460, 92159807707754167187997289514579132865, 99249023685273718510150927169407637270, 127605887595351923798765477788721654890, 134695103572871475120919115443550159295, 141784319550391026443072753098378663700",
      "INFO [AntiEntropySessions:1] 2013-04-24 23:21:16,587 AntiEntropyService.java (line 651) repair #a9a87e40-ad35-11e2-945a-050d956ff11b new session: will sync /YYY.XX.98.11, /YY.XXX.107.137, /YY.XXX.133.163 on range (99249023685273718510150927169407637270,127605887595351923798765477788721654890] for cardspring_production.[App]",
      "INFO [AntiEntropySessions:1] 2013-04-24 23:21:16,598 AntiEntropyService.java (line 857) repair #a9a87e40-ad35-11e2-945a-050d956ff11b requesting merkle trees for App (to [/XX.YYY.107.137, /XX.YYY.133.163, /XXX.YY.98.11])",
      "DEBUG [WRITE-/107.20.98.11] 2013-04-24 23:21:16,601 OutboundTcpConnection.java (line 260) attempting to connect to /XXX.YY.98.11",
      "INFO [AntiEntropyStage:1] 2013-04-24 23:21:19,111 AntiEntropyService.java (line 213) repair #a9a87e40-ad35-11e2-945a-050d956ff11b Received merkle tree for App from /XX.YYY.133.163",
      "DEBUG [ScheduledTasks:1] 2013-04-24 23:21:19,409 GCInspector.java (line 121) GC for ParNew: 54 ms for 1 collections, 669806384 used; max is 4211081216",
      "INFO [AntiEntropyStage:1] 2013-04-24 23:21:20,408 AntiEntropyService.java (line 213) repair #a9a87e40-ad35-11e2-945a-050d956ff11b Received merkle tree for App from /XX.YYY.107.137"
    ],
    "stack_traces": []
  },
  {
    "file": "Cassandra-5529.docx",
    "title": "Cassandra-5529",
    "version": "cassandra-1.1.11",
    "Description": "thrift_max_message_length_in_mb makes long-lived connections error out",
    "logs": [
      "WARN org.apache.hadoop.mapred.Child: Error running child",
      "INFO org.apache.hadoop.mapred.Task: Runnning cleanup for the task"
    ],
    "stack_traces": [
      "java.lang.RuntimeException: com.rockmelt.org.apache.thrift.TException: Message length exceeded: 40\nat org.apache.cassandra.hadoop.ColumnFamilyRecordReader$StaticRowIterator.maybeInit(ColumnFamilyRecordReader.java:400)\nat org.apache.cassandra.hadoop.ColumnFamilyRecordReader$StaticRowIterator.computeNext(ColumnFamilyRecordReader.java:406)\nat org.apache.cassandra.hadoop.ColumnFamilyRecordReader$StaticRowIterator.computeNext(ColumnFamilyRecordReader.java:329)\nat com.google.common.collect.AbstractIterator.tryToComputeNext(AbstractIterator.java:143)\nat com.google.common.collect.AbstractIterator.hasNext(AbstractIterator.java:138)\nat org.apache.cassandra.hadoop.ColumnFamilyRecordReader.getProgress(ColumnFamilyRecordReader.java:109)\nat org.apache.hadoop.mapred.MapTask$NewTrackingRecordReader.getProgress(MapTask.java:522)\nat org.apache.hadoop.mapred.MapTask$NewTrackingRecordReader.nextKeyValue(MapTask.java:547)\nat org.apache.hadoop.mapreduce.MapContext.nextKeyValue(MapContext.java:67)\nat org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:143)\nat org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:771)\nat org.apache.hadoop.mapred.MapTask.run(MapTask.java:375)\nat org.apache.hadoop.mapred.Child$4.run(Child.java:255)\nat java.security.AccessController.doPrivileged(Native Method)\nat javax.security.auth.Subject.doAs(Subject.java:396)\nat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1132)\nat org.apache.hadoop.mapred.Child.main(Child.java:249)\nCaused by: com.rockmelt.org.apache.thrift.TException: Message length exceeded: 40\nat com.rockmelt.org.apache.thrift.protocol.TBinaryProtocol.checkReadLength(TBinaryProtocol.java:393)\nat com.rockmelt.org.apache.thrift.protocol.TBinaryProtocol.readBinary(TBinaryProtocol.java:363)\nat org.apache.cassandra.thrift.Column.read(Column.java:528)\nat org.apache.cassandra.thrift.ColumnOrSuperColumn.read(ColumnOrSuperColumn.java:507)\nat org.apache.cassandra.thrift.KeySlice.read(KeySlice.java:408)\nat org.apache.cassandra.thrift.Cassandra$get_range_slices_result.read(Cassandra.java:12422)\nat com.rockmelt.org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:78)\nat org.apache.cassandra.thrift.Cassandra$Client.recv_get_range_slices(Cassandra.java:696)\nat org.apache.cassandra.thrift.Cassandra$Client.get_range_slices(Cassandra.java:680)\nat org.apache.cassandra.hadoop.ColumnFamilyRecordReader$StaticRowIterator.maybeInit(ColumnFamilyRecordReader.java:362)\n... 16 more"
    ]
  },
  {
    "file": "Cassandra-5631.docx",
    "title": "Cassandra-5631",
    "version": "cassandra-1.2.15",
    "Description": "NPE when creating column family shortly after multinode startup",
    "logs": [
      "2013-06-12 14:55:31,773 ERROR CassandraDaemon [MigrationStage:1] - Exception in thread Thread[MigrationStage:1,5,main]",
      "2013-06-12 14:55:31,880 ERROR CassandraDaemon [MigrationStage:1] - Exception in thread Thread[MigrationStage:1,5,main]"
    ],
    "stack_traces": [
      "java.lang.NullPointerException\n\t\t\t\t\tat org.apache.cassandra.db.DefsTable.addColumnFamily(DefsTable.java:510)\n\t\t\t\t\tat org.apache.cassandra.db.DefsTable.mergeColumnFamilies(DefsTable.java:444)\n\t\t\t\t\tat org.apache.cassandra.db.DefsTable.mergeSchema(DefsTable.java:354)\n\t\t\t\t\tat org.apache.cassandra.db.DefinitionsUpdateVerbHandler$1.runMayThrow(DefinitionsUpdateVerbHandler.java:55)\n\t\t\t\t\tat org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)\n\t\t\t\t\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)\n\t\t\t\t\tat java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)\n\t\t\t\t\tat java.util.concurrent.FutureTask.run(FutureTask.java:166)\n\t\t\t\t\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n\t\t\t\t\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\t\t\t\t\tat java.lang.Thread.run(Thread.java:722)",
      "java.lang.NullPointerException\n\t\t\t\t\tat org.apache.cassandra.db.DefsTable.mergeColumnFamilies(DefsTable.java:475)\n\t\t\t\t\tat org.apache.cassandra.db.DefsTable.mergeSchema(DefsTable.java:354)\n\t\t\t\t\tat org.apache.cassandra.db.DefinitionsUpdateVerbHandler$1.runMayThrow(DefinitionsUpdateVerbHandler.java:55)\n\t\t\t\t\tat org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)\n\t\t\t\t\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)\n\t\t\t\t\tat java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)\n\t\t\t\t\tat java.util.concurrent.FutureTask.run(FutureTask.java:166)\n\t\t\t\t\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n\t\t\t\t\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\t\t\t\t\tat java.lang.Thread.run(Thread.java:722)"
    ]
  },
  {
    "file": "Cassandra-5701.docx",
    "title": "Cassandra-5701",
    "version": "cassandra-2.0.4",
    "Description": "Apache.Cassandra.Cassandra.get_count will disconnect but not throw InvalidRequestException when column family is not existed.",
    "logs": [
      "ERROR [RPC-Thread:3373] 2013-06-26 14:23:09,264 TNonblockingServer.java (line 638) Unexpected exception while invoking!"
    ],
    "stack_traces": [
      "java.lang.IllegalArgumentException: Unknown table/cf pair (Keyspace1.Standard)\nat org.apache.cassandra.db.Table.getColumnFamilyStore(Table.java:165)\nat org.apache.cassandra.thrift.CassandraServer.get_count(CassandraServer.java:471)\nat org.apache.cassandra.thrift.Cassandra$Processor$get_count.getResult(Cassandra.java:3381)\nat org.apache.cassandra.thrift.Cassandra$Processor$get_count.getResult(Cassandra.java:3369)\nat org.apache.thrift.ProcessFunction.process(ProcessFunction.java:32)\nat org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:34)\nat org.apache.thrift.server.TNonblockingServer$FrameBuffer.invoke(TNonblockingServer.java:632)\nat org.apache.cassandra.thrift.CustomTHsHaServer$Invocation.run(CustomTHsHaServer.java:109)\nat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(Unknown Source)\nat java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)\nat java.lang.Thread.run(Unknown Source)"
    ]
  },
  {
    "file": "Cassandra-5725.docx",
    "title": "Cassandra-5725",
    "version": "cassandra-1.2.10",
    "Description": "Silently failing messages in case of schema not fully propagated",
    "logs": [
      "INFO 13:11:39,441 IOException reading from socket; closing"
    ],
    "stack_traces": [
      "org.apache.cassandra.db.UnknownColumnFamilyException: Couldn't find cfId=a31c7604-0e40-393b-82d7-ba3d910ad50a\n\t\tat org.apache.cassandra.db.ColumnFamilySerializer.deserializeCfId(ColumnFamilySerializer.java:184)\n\t\tat org.apache.cassandra.db.ColumnFamilySerializer.deserialize(ColumnFamilySerializer.java:94)\n\t\tat org.apache.cassandra.db.RowMutation$RowMutationSerializer.deserialize(RowMutation.java:397)\n\t\tat org.apache.cassandra.db.RowMutation$RowMutationSerializer.deserialize(RowMutation.java:407)\n\t\tat org.apache.cassandra.db.RowMutation$RowMutationSerializer.deserialize(RowMutation.java:367)\n\t\tat org.apache.cassandra.net.MessageIn.read(MessageIn.java:94)\n\t\tat org.apache.cassandra.net.IncomingTcpConnection.receiveMessage(IncomingTcpConnection.java:207)\n\t\tat org.apache.cassandra.net.IncomingTcpConnection.handleModernVersion(IncomingTcpConnection.java:139)\n\t\tat org.apache.cassandra.net.IncomingTcpConnection.run(IncomingTcpConnection.java:82)"
    ]
  },
  {
    "file": "Cassandra-5804.docx",
    "title": "Cassandra-5804",
    "version": "cassandra-1.2.7",
    "Description": "AntiEntropySession fails when OutboundTcpConnection receives IOException",
    "logs": [
      "INFO [AntiEntropySessions:5] 2013-07-24 20:16:39,232 AntiEntropyService.java (line 651) [repair #79afee40-f4bf-11e2-bfb6-bd4071a4c32e] new session: will sync /192.168.1.93, /192.168.2.92, /192.168.2.91, /192.168.1.91 on range (6575400599453278172,6596229519918600663) for ks1.[cf1, cf2]",
      "INFO [AntiEntropySessions:5] 2013-07-24 20:16:39,233 AntiEntropyService.java (line 857) [repair #79afee40-f4bf-11e2-bfb6-bd4071a4c32e] requesting merkle trees for cf1 (to [/192.168.2.92, /192.168.2.91, /192.168.1.91, /192.168.1.93])",
      "TRACE [AntiEntropySessions:5] 2013-07-24 20:16:39,233 MessagingService.java (line 602) /192.168.1.93 sending TREE_REQUEST to 320929@/192.168.2.91",
      "TRACE [AntiEntropySessions:5] 2013-07-24 20:16:39,233 MessagingService.java (line 602) /192.168.1.93 sending TREE_REQUEST to 320930@/192.168.1.93",
      "TRACE [AntiEntropySessions:5] 2013-07-24 20:16:39,234 MessagingService.java (line 605) Message-to-self TYPE:ANTI_ENTROPY VERB:TREE_REQUEST going over MessagingService",
      "TRACE [AntiEntropySessions:5] 2013-07-24 20:16:39,235 MessagingService.java (line 602) /192.168.1.93 sending TREE_REQUEST to 320931@/192.168.2.92",
      "TRACE [AntiEntropySessions:5] 2013-07-24 20:16:39,235 MessagingService.java (line 602) /192.168.1.93 sending TREE_REQUEST to 320932@/192.168.1.91",
      "DEBUG [AntiEntropyStage:1] 2013-07-24 20:16:39,236 AntiEntropyService.java (line 467) Queueing validation compaction for #<TreeRequest 79afee40-f4bf-11e2-bfb6-bd4071a4c32e, /192.168.1.93, (ks1,cf1), (6575400599453278172,6596229519918600663)>",
      "DEBUG [WRITE-/192.168.2.92] 2013-07-24 20:16:39,237 OutboundTcpConnection.java (line 209) error writing to /192.168.2.92",
      "DEBUG [ValidationExecutor:3] 2013-07-24 20:16:39,239 StorageService.java (line 2331) Forcing flush on keyspace ks1, CF cf1",
      "DEBUG [WRITE-/192.168.2.91] 2013-07-24 20:16:39,237 OutboundTcpConnection.java (line 209) error writing to /192.168.2.91"
    ],
    "stack_traces": [
      "java.io.IOException: Connection reset by peer\n\t\tat sun.nio.ch.FileDispatcher.write0(Native Method)\n\t\tat sun.nio.ch.SocketDispatcher.write(Unknown Source)\n\t\tat sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown Source)\n\t\tat sun.nio.ch.IOUtil.write(Unknown Source)\n\t\tat sun.nio.ch.SocketChannelImpl.write(Unknown Source)\n\t\tat java.nio.channels.Channels.writeFullyImpl(Unknown Source)\n\t\tat java.nio.channels.Channels.writeFully(Unknown Source)\n\t\tat java.nio.channels.Channels.access$000(Unknown Source)\n\t\tat java.nio.channels.Channels$1.write(Unknown Source)\n\t\tat java.io.BufferedOutputStream.flushBuffer(Unknown Source)\n\t\tat java.io.BufferedOutputStream.flush(Unknown Source)\n\t\tat org.xerial.snappy.SnappyOutputStream.flush(SnappyOutputStream.java:272)\n\t\tat java.io.DataOutputStream.flush(Unknown Source)\n\t\tat org.apache.cassandra.net.OutboundTcpConnection.writeConnected(OutboundTcpConnection.java:200)\n\t\tat org.apache.cassandra.net.OutboundTcpConnection.run(OutboundTcpConnection.java:144)",
      "java.io.IOException: Connection reset by peer\n\t\tat sun.nio.ch.FileDispatcher.write0(Native Method)\n\t\tat sun.nio.ch.SocketDispatcher.write(Unknown Source)\n\t\tat sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown Source)\n\t\tat sun.nio.ch.IOUtil.write(Unknown Source)\n\t\tat sun.nio.ch.SocketChannelImpl.write(Unknown Source)\n\t\tat java.nio.channels.Channels.writeFullyImpl(Unknown Source)\n\t\tat java.nio.channels.Channels.writeFully(Unknown Source)\n\t\tat java.nio.channels.Channels.access$000(Unknown Source)\n\t\tat java.nio.channels.Channels$1.write(Unknown Source)\n\t\tat java.io.BufferedOutputStream.flushBuffer(Unknown Source)\n\t\tat java.io.BufferedOutputStream.flush(Unknown Source)\n\t\tat org.xerial.snappy.SnappyOutputStream.flush(SnappyOutputStream.java:272)\n\t\tat java.io.DataOutputStream.flush(Unknown Source)\n\t\tat org.apache.cassandra.net.OutboundTcpConnection.writeConnected(OutboundTcpConnection.java:200)\n\t\tat org.apache.cassandra.net.OutboundTcpConnection.run(OutboundTcpConnection.java:144)"
    ]
  },
  {
    "file": "Cassandra-6098.docx",
    "title": "Cassandra-6098",
    "version": "cassandra-2.0.1",
    "Description": "NullPointerException causing query timeout",
    "logs": [
      "ERROR 15:38:04,036 Exception in thread Thread[ReadStage:170,5,main]"
    ],
    "stack_traces": [
      "java.lang.RuntimeException: java.lang.NullPointerException\n\t\tat org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1867)\n\t\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n\t\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\t\tat java.lang.Thread.run(Thread.java:724)\nCaused by: java.lang.NullPointerException\n\t\tat org.apache.cassandra.db.index.composites.CompositesIndexOnRegular.isStale(CompositesIndexOnRegular.java:97)\n\t\tat org.apache.cassandra.db.index.composites.CompositesSearcher$1.computeNext(CompositesSearcher.java:247)\n\t\tat org.apache.cassandra.db.index.composites.CompositesSearcher$1.computeNext(CompositesSearcher.java:102)\n\t\tat com.google.common.collect.AbstractIterator.tryToComputeNext(AbstractIterator.java:143)\n\t\tat com.google.common.collect.AbstractIterator.hasNext(AbstractIterator.java:138)\n\t\tat org.apache.cassandra.db.ColumnFamilyStore.filter(ColumnFamilyStore.java:1651)\n\t\tat org.apache.cassandra.db.index.composites.CompositesSearcher.search(CompositesSearcher.java:50)\n\t\tat org.apache.cassandra.db.index.SecondaryIndexManager.search(SecondaryIndexManager.java:525)\n\t\tat org.apache.cassandra.db.ColumnFamilyStore.search(ColumnFamilyStore.java:1639)\n\t\tat org.apache.cassandra.db.RangeSliceCommand.executeLocally(RangeSliceCommand.java:135)\n\t\tat org.apache.cassandra.service.StorageProxy$LocalRangeSliceRunnable.runMayThrow(StorageProxy.java:1358)\n\t\tat org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1863)"
    ]
  },
  {
    "file": "Cassandra-6169.docx",
    "title": "Cassandra-6169",
    "version": "cassandra-1.2.10",
    "Description": "Too many splits cause \"OutOfMemoryError: unable to create new native thread\" in AbstractColumnFamilyInputFormat",
    "logs": [],
    "stack_traces": [
      "java.lang.OutOfMemoryError: unable to create new native thread\n\t\t\t\t\tat java.lang.Thread.start0(Native Method)\n\t\t\t\t\tat java.lang.Thread.start(Thread.java:691)\n\t\t\t\t\tat java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:943)\n\t\t\t\t\tat java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1336)\n\t\t\t\t\tat java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:132)\n\t\t\t\t\tat org.apache.cassandra.hadoop.AbstractColumnFamilyInputFormat.getSplits(AbstractColumnFamilyInputFormat.java:187)\n\t\t\t\t\tat org.apache.hadoop.mapred.JobClient.writeNewSplits(JobClient.java:1054)\n\t\t\t\t\tat org.apache.hadoop.mapred.JobClient.writeSplits(JobClient.java:1071)\n\t\t\t\t\tat org.apache.hadoop.mapred.JobClient.access$700(JobClient.java:179)\n\t\t\t\t\tat org.apache.hadoop.mapred.JobClient$2.run(JobClient.java:983)\n\t\t\t\t\tat org.apache.hadoop.mapred.JobClient$2.run(JobClient.java:936)\n\t\t\t\t\tat java.security.AccessController.doPrivileged(Native Method)\n\t\t\t\t\tat javax.security.auth.Subject.doAs(Subject.java:415)\n\t\t\t\t\tat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1190)\n\t\t\t\t\tat org.apache.hadoop.mapred.JobClient.submitJobInternal(JobClient.java:936)\n\t\t\t\t\tat org.apache.hadoop.mapreduce.Job.submit(Job.java:550)\n\t\t\t\t\tat org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:580)\n\t\t\t\t\tat com.relateiq.hadoop.cassandra.etl.CassandraETLJob.run(CassandraETLJob.java:58)\n\t\t\t\t\tat org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:65)\n\t\t\t\t\tat com.relateiq.hadoop.cassandra.etl.CassandraETLJob.main(CassandraETLJob.java:149)"
    ]
  },
  {
    "file": "Cassandra-6622.docx",
    "title": "Cassandra-6622",
    "version": "cassandra-2.0.4",
    "Description": "Streaming session failures during node replace of same address",
    "logs": [
      "TRACE [GossipTasks:1] 2014-02-08 18:55:23,656 Gossiper.java (line 598) Performing status check",
      "TRACE [GossipTasks:1] 2014-02-08 18:55:23,657 FailureDetector.java (line 229) PHI for /x.x.x.72 : 36643.639468390524",
      "TRACE [GossipTasks:1] 2014-02-08 18:55:23,657 FailureDetector.java (line 233) notifying listeners that /x.x.x.72 is down",
      "TRACE [GossipTasks:1] 2014-02-08 18:55:24,659 Gossiper.java (line 598) Performing status check ...",
      "TRACE [GossipTasks:1] 2014-02-08 18:55:24,660 FailureDetector.java (line 229) PHI for /x.x.x.72 : 36644.432127252854",
      "TRACE [GossipTasks:1] 2014-02-08 18:55:24,660 FailureDetector.java (line 233) notifying listeners that /x.x.x.72 is down",
      "TRACE [GossipTasks:1] 2014-02-08 18:55:25,662 Gossiper.java (line 598) Performing status check ...",
      "TRACE [GossipTasks:1] 2014-02-08 18:55:25,662 FailureDetector.java (line 229) PHI for /x.x.x.72 : 36645.22442135811",
      "TRACE [GossipTasks:1] 2014-02-08 18:55:25,663 FailureDetector.java (line 233) notifying listeners that /x.x.x.72 is down",
      "TRACE [GossipTasks:1] 2014-02-08 18:55:26,664 Gossiper.java (line 598) Performing status check ...",
      "TRACE [GossipTasks:1] 2014-02-08 18:55:26,665 FailureDetector.java (line 229) PHI for /x.x.x.72 : 36646.016701132285",
      "TRACE [GossipTasks:1] 2014-02-08 18:55:26,665 FailureDetector.java (line 233) notifying listeners that /x.x.x.72 is down",
      "INFO [main] 2014-02-08 18:56:27,806 CassandraDaemon.java (line 130) Logging initialized [node x.x.x.72, replaced with same IP]",
      "INFO [main] 2014-02-08 18:56:27,833 YamlConfigurationLoader.java (line 76) Loading settings from file:/home/y/libexec64/cassandra/conf/cassandra.yaml [node x.x.x.72]",
      "INFO [main] 2014-02-08 18:56:33,308 StorageService.java (line 627) Starting up server gossip [node x.x.x.72]",
      "INFO [main] 2014-02-08 18:56:35,405 StorageService.java (line 947) JOINING: Starting to bootstrap... [node x.x.x.72]",
      "INFO [CompactionExecutor:1] 2014-02-08 18:56:35,432 CompactionTask.java (line 275) Compacted 4 sstables to [/home/y/var/cassandra/data/system/local/system-local-jb-5,]. 755 bytes to 538 (~71% of original) in 25ms = 0.020523MB/s. 4 total partitions merged to 1. Partition merge counts were {4:1, } [node x.x.x.72]",
      "INFO [main] 2014-02-08 18:56:35,443 StreamResultFuture.java (line 82) [Stream #bb897500-90f2-11e3-9d67-d5d417af8653] Executing streaming plan for Bootstrap [node x.x.x.72]",
      "INFO [main] 2014-02-08 18:56:35,443 StreamResultFuture.java (line 86) [Stream #bb897500-90f2-11e3-9d67-d5d417af8653] Beginning stream session with /x.x.x.80 [node x.x.x.72]",
      "INFO [main] 2014-02-08 18:56:35,444 StreamResultFuture.java (line 86) [Stream #bb897500-90f2-11e3-9d67-d5d417af8653] Beginning stream session with /x.x.x.81 [node x.x.x.72]",
      "INFO [main] 2014-02-08 18:56:35,444 StreamResultFuture.java (line 86) [Stream #bb897500-90f2-11e3-9d67-d5d417af8653] Beginning stream session with /x.x.x.73 [node x.x.x.72]",
      "INFO [STREAM-IN-/x.x.x.81] 2014-02-08 18:56:35,462 StreamResultFuture.java (line 181) [Stream #bb897500-90f2-11e3-9d67-d5d417af8653] Session with /x.x.x.81 is complete [node x.x.x.72]",
      "INFO [STREAM-INIT-/x.x.x.72:47408] 2014-02-08 18:56:35,450 StreamResultFuture.java (line 116) [Stream #bb897500-90f2-11e3-9d67-d5d417af8653] Received streaming plan for Bootstrap [node x.x.x.80]",
      "INFO [STREAM-IN-/x.x.x.72] 2014-02-08 18:56:35,460 ColumnFamilyStore.java (line 784) Enqueuing flush of Memtable-facetrevs4@509747825(251160/2013133 serialized/live bytes, 20934 ops) [node x.x.x.80]",
      "TRACE [GossipTasks:1] 2014-02-08 18:56:36,090 FailureDetector.java (line 229) PHI for /x.x.x.72 : 36700.87918907657 [node x.x.x.80]",
      "TRACE [GossipTasks:1] 2014-02-08 18:56:36,090 FailureDetector.java (line 233) notifying listeners that /x.x.x.72 is down [node x.x.x.80]",
      "INFO [GossipTasks:1] 2014-02-08 18:56:36,090 StreamResultFuture.java (line 181) [Stream #bb897500-90f2-11e3-9d67-d5d417af8653] Session with /x.x.x.72 is complete [node x.x.x.80]",
      "WARN [GossipTasks:1] 2014-02-08 18:56:36,091 StreamResultFuture.java (line 210) [Stream #bb897500-90f2-11e3-9d67-d5d417af8653] Stream failed [node x.x.x.80]",
      "DEBUG [GossipStage:1] 2014-02-08 18:56:36,097 Gossiper.java (line 790) Clearing interval times for /x.x.x.72 due to generation change [node x.x.x.80]",
      "TRACE [GossipStage:1] 2014-02-08 18:56:36,097 FailureDetector.java (line 203) reporting /x.x.x.72 [node x.x.x.80]",
      "TRACE [GossipStage:1] 2014-02-08 18:56:36,097 Gossiper.java (line 946) /x.x.x.72local generation 1391830955, remote generation 1391885793 [node x.x.x.80]",
      "TRACE [GossipStage:1] 2014-02-08 18:56:36,097 Gossiper.java (line 951) Updating heartbeat state generation to 1391885793 from 1391830955 for /x.x.x.72 [node x.x.x.80]",
      "TRACE [GossipStage:1] 2014-02-08 18:56:36,097 Gossiper.java (line 886) Adding endpoint state for /x.x.x.72 [node x.x.x.80]",
      "TRACE [GossipTasks:1] 2014-02-08 18:56:37,093 Gossiper.java (line 598) Performing status check ... [node x.x.x.80]",
      "TRACE [GossipTasks:1] 2014-02-08 18:56:37,094 FailureDetector.java (line 229) PHI for /x.x.x.72 : 0.06483452387313912 [node x.x.x.80]",
      "TRACE [GossipTasks:1] 2014-02-08 18:56:37,094 FailureDetector.java (line 229) PHI for /x.x.x.81 : 0.7806744414980482 [node x.x.x.80]",
      "TRACE [GossipTasks:1] 2014-02-08 18:56:37,094 FailureDetector.java (line 229) PHI for /x.x.x.71 : 1.7235166979455117 [node x.x.x.80]",
      "TRACE [GossipTasks:1] 2014-02-08 18:56:37,094 FailureDetector.java (line 229) PHI for /x.x.x.84 : 0.8088014574340049 [node x.x.x.80]",
      "ERROR [STREAM-IN-/x.x.x.72] 2014-02-08 18:56:37,105 StreamSession.java (line 418) [Stream #bb897500-90f2-11e3-9d67-d5d417af8653] Streaming error occurred [node x.x.x.80]",
      "INFO [STREAM-IN-/x.x.x.72] 2014-02-08 18:56:37,108 StreamResultFuture.java (line 181) [Stream #bb897500-90f2-11e3-9d67-d5d417af8653] Session with /x.x.x.72 is complete [node x.x.x.80]",
      "WARN [STREAM-IN-/x.x.x.72] 2014-02-08 18:56:37,108 StreamResultFuture.java (line 210) [Stream #bb897500-90f2-11e3-9d67-d5d417af8653] Stream failed [node x.x.x.80]"
    ],
    "stack_traces": [
      "java.lang.RuntimeException: Outgoing stream handler has been closed\n\t\t\t\t\tat org.apache.cassandra.streaming.ConnectionHandler.sendMessage(ConnectionHandler.java:170)\n\t\t\t\t\tat org.apache.cassandra.streaming.StreamSession.prepare(StreamSession.java:444)\n\t\t\t\t\tat org.apache.cassandra.streaming.StreamSession.messageReceived(StreamSession.java:366)\n\t\t\t\t\tat org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:289)\n\t\t\t\t\tat java.lang.Thread.run(Thread.java:724)"
    ]
  },
  {
    "file": "HBase-10210.docx",
    "title": "HBase-10210",
    "version": "hbase-0.98.0",
    "Description": "during master startup, RS can be you-are-dead-ed by master in error",
    "logs": [
      "2013-12-19 11:16:45,290 INFO [master:h2-ubuntu12-sec-1387431063-hbase-10:60000] master.ServerManager: Finished waiting for region servers count to settle; checked in 2, slept for 18262 ms, expecting minimum of 1, maximum of 2147483647, master is running.",
      "2013-12-19 11:16:45,290 INFO [master:h2-ubuntu12-sec-1387431063-hbase-10:60000] master.ServerManager: Registering server=h2-ubuntu12-sec-1387431063-hbase-8.cs1cloud.internal,60020,1387451803800",
      "2013-12-19 11:16:45,290 INFO [master:h2-ubuntu12-sec-1387431063-hbase-10:60000] master.HMaster: Registered server found up in zk but who has not yet reported in: h2-ubuntu12-sec-1387431063-hbase-8.cs1cloud.internal,60020,1387451803800",
      "2013-12-19 11:16:45,380 INFO [RpcServer.handler=4,port=60000] master.ServerManager: Triggering server recovery; existingServer h2-ubuntu12-sec-1387431063-hbase-8.cs1cloud.internal,60020,1387451803800 looks stale, new server:h2-ubuntu12-sec-1387431063-hbase-8.cs1cloud.internal,60020,1387451803800",
      "2013-12-19 11:16:45,380 INFO [RpcServer.handler=4,port=60000] master.ServerManager: Master doesn't enable ServerShutdownHandler during initialization, delay expiring server h2-ubuntu12-sec-1387431063-hbase-8.cs1cloud.internal,60020,1387451803800",
      "2013-12-19 11:16:46,925 ERROR [RpcServer.handler=7,port=60000] master.HMaster: Region server h2-ubuntu12-sec-1387431063-hbase-8.cs1cloud.internal,60020,1387451803800 reported a fatal error: ABORTING region server h2-ubuntu12-sec-1387431063-hbase-8.cs1cloud.internal,60020,1387451803800: org.apache.hadoop.hbase.YouAreDeadException: Server REPORT rejected; currently processing h2-ubuntu12-sec-1387431063-hbase-8.cs1cloud.internal,60020,1387451803800 as dead server"
    ],
    "stack_traces": []
  },
  {
    "file": "HBase-10214.docx",
    "title": "HBase-10214",
    "version": "hbase-0.94.14",
    "Description": "Regionserver shutdown improperly and leaves the dir in .old not deleted",
    "logs": [
      "2013-12-18 15:17:45,771 DEBUG org.apache.hadoop.hbase.regionserver.HRegionServer: Waiting on 51b27391410efdca841db264df46085f",
      "2013-12-18 15:17:45,776 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: Connected to master at null",
      "2013-12-18 15:17:48,776 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: STOPPED: Exiting; cluster shutdown set and not carrying any regions",
      "2013-12-18 15:17:48,776 FATAL org.apache.hadoop.hbase.regionserver.HRegionServer: ABORTING region server node,60020,1384410974572: Unhandled exception: null"
    ],
    "stack_traces": [
      "java.lang.NullPointerException\n\t\tat org.apache.hadoop.hbase.regionserver.HRegionServer.tryRegionServerReport(HRegionServer.java:880)\n\t\tat org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:753)\n\t\tat java.lang.Thread.run(Thread.java:662)"
    ]
  },
  {
    "file": "HBase-3617.docx",
    "title": "HBase-3617",
    "version": "hbase-0.90.3",
    "Description": "NoRouteToHostException during balancing will cause Master abort",
    "logs": [
      "2011-03-10 07:48:39,192 FATAL org.apache.hadoop.hbase.master.HMaster: Remote unexpected exception",
      "2011-03-10 07:48:39,192 INFO org.apache.hadoop.hbase.master.HMaster: Aborting",
      "2011-03-10 07:48:39,192 INFO org.apache.hadoop.hbase.master.HMaster: balance hri=SpecialObject_Speed_Test,,1299710751983.f0e5544339870a510c338b3029979d3e.,src=ap13.secur2,60020,1299710609447,dest=ap12.secur2,60020,1299710609148",
      "2011-03-10 07:48:39,192 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Starting unassignment of region SpecialObject_Speed_Test,, 1299710751983.f0e5544339870a510c338b3029979d3e. (offlining)",
      "2011-03-10 07:48:39,852 DEBUG org.apache.hadoop.hbase.master.HMaster: Stopping service threads",
      "2011-03-10 07:48:39,852 INFO org.apache.hadoop.ipc.HBaseServer: Stopping server on 60000",
      "2011-03-10 07:48:39,852 FATAL org.apache.hadoop.hbase.master.HMaster: Remote unexpected exception",
      "2011-03-10 07:48:39,852 INFO org.apache.hadoop.hbase.master.HMaster: Aborting"
    ],
    "stack_traces": [
      "java.net.NoRouteToHostException: No route to host\n\tat sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)\n\tat sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:567)\n\tat org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)\n\tat org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)\n\tat org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)\n\tat org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)\n\tat org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)\n\tat org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)\n\tat $Proxy6.closeRegion(Unknown Source)\n\tat org.apache.hadoop.hbase.master.ServerManager.sendRegionClose(ServerManager.java:589)\n\tat org.apache.hadoop.hbase.master.AssignmentManager.unassign(AssignmentManager.java:1093)\n\tat org.apache.hadoop.hbase.master.AssignmentManager.unassign(AssignmentManager.java:1040)\n\tat org.apache.hadoop.hbase.master.AssignmentManager.balance(AssignmentManager.java:1831)\n\tat org.apache.hadoop.hbase.master.HMaster.balance(HMaster.java:692)\n\tat org.apache.hadoop.hbase.master.HMaster$1.chore(HMaster.java:583)\n\tat org.apache.hadoop.hbase.Chore.run(Chore.java:66)",
      "java.io.InterruptedIOException: Interruped while waiting for IO on channel java.nio.channels.SocketChannel[connection-pending remote=/10.X.X.18:60020]. 19340 millis timeout left.\n\tat org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:349)\n\tat org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:203)\n\tat org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)\n\tat org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)\n\tat org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)\n\tat org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)\n\tat org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)\n\tat $Proxy6.closeRegion(Unknown Source)\n\tat org.apache.hadoop.hbase.master.ServerManager.sendRegionClose(ServerManager.java:589)\n\tat org.apache.hadoop.hbase.master.AssignmentManager.unassign(AssignmentManager.java:1093)\n\tat org.apache.hadoop.hbase.master.AssignmentManager.unassign(AssignmentManager.java:1040)\n\tat org.apache.hadoop.hbase.master.AssignmentManager.balance(AssignmentManager.java:1831)\n\tat org.apache.hadoop.hbase.master.HMaster.balance(HMaster.java:692)\n\tat org.apache.hadoop.hbase.master.HMaster$1.chore(HMaster.java:583)\n\tat org.apache.hadoop.hbase.Chore.run(Chore.java:66)"
    ]
  },
  {
    "file": "HBase-3722.docx",
    "title": "HBase-3722",
    "version": "hbase-0.90.1",
    "Description": "A lot of data is lost when name node crashed",
    "logs": [
      "2011-03-22 13:21:55,056 WARN org.apache.hadoop.hbase.master.LogCleaner: Error while cleaning the logs",
      "2011-03-22 13:21:56,056 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: C4C1/157.5.100.1:9000. Already tried 0 time(s).",
      "2011-03-22 13:21:57,057 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: C4C1/157.5.100.1:9000. Already tried 1 time(s).",
      "2011-03-22 13:22:05,060 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: C4C1/157.5.100.1:9000. Already tried 9 time(s).",
      "2011-03-22 13:22:05,060 ERROR org.apache.hadoop.hbase.master.MasterFileSystem: Failed splitting hdfs://C4C1:9000/hbase/.logs/C4C9.site,60020,1300767633398",
      "2011-03-22 13:22:45,600 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: C4C1/157.5.100.1:9000. Already tried 0 time(s).",
      "2011-03-22 13:22:46,600 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: C4C1/157.5.100.1:9000. Already tried 1 time(s).",
      "2011-03-22 13:22:54,603 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: C4C1/157.5.100.1:9000. Already tried 9 time(s).",
      "2011-03-22 13:22:54,603 WARN org.apache.hadoop.hbase.master.LogCleaner: Error while cleaning the logs",
      "2011-04-12 14:57:58 C4C3 shutdow itself because of ANN crashed. skip splitlog and ressigned Meta table.",
      "2011-04-12 14:57:58,782 INFO org.apache.hadoop.hbase.master.handler.ServerShutdownHandler: Splitting logs for C4C3.site,60020,1302590910433",
      "2011-04-12 14:57:59,790 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: C4C1/157.5.100.1:9000. Already tried 0 time(s).",
      "2011-04-12 14:58:08,793 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: C4C1/157.5.100.1:9000. Already tried 9 time(s).",
      "2011-04-12 14:58:08,795 ERROR org.apache.hadoop.hbase.master.MasterFileSystem: Failed splitting hdfs://C4C1:9000/hbase/.logs/C4C3.site,60020,1302590910433",
      "2011-04-12 14:58:08,805 INFO org.apache.hadoop.hbase.catalog.RootLocationEditor: Unsetting ROOT region location in ZooKeeper",
      "2011-04-12 14:58:08,880 INFO org.apache.hadoop.hbase.catalog.CatalogTracker: Failed verification of .META.,,1 at address=C4C3.site:60020; java.net.ConnectException: Connection refused",
      "2011-04-12 14:58:08,880 INFO org.apache.hadoop.hbase.catalog.CatalogTracker: Current cached META location is not valid, resetting",
      "2011-04-12 15:00:31,681 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: C4C1/157.5.100.1:9000. Already tried 0 time(s).",
      "2011-04-12 15:00:32,682 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: C4C1/157.5.100.1:9000. Already tried 1 time(s).",
      "2011-04-12 15:00:40,698 INFO org.apache.hadoop.hbase.master.AssignmentManager: Regions in transition timed out: .META.,,1.1028785192 state=OPENING, ts=1302591600701",
      "2011-04-12 15:00:40,699 INFO org.apache.hadoop.hbase.master.AssignmentManager: Region has been OPENING for too long, reassigning region=.META.,,1.1028785192",
      "2011-04-12 15:00:40,709 INFO org.apache.hadoop.hbase.master.AssignmentManager: Successfully transitioned region=.META.,,1.1028785192 into OFFLINE and forcing a new assignment",
      "2011-04-12 15:00:40,712 INFO org.apache.hadoop.hbase.master.AssignmentManager: Regions in transition timed out: ROOT,,0.70236052 state=OPENING, ts=1302591600718",
      "2011-04-12 15:00:40,712 INFO org.apache.hadoop.hbase.master.AssignmentManager: Region has been OPENING for too long, reassigning region=ROOT,,0.70236052",
      "2011-04-12 15:00:40,725 INFO org.apache.hadoop.hbase.master.AssignmentManager: Successfully transitioned region=ROOT,,0.70236052 into OFFLINE and forcing a new assignment",
      "2011-04-12 15:00:40,892 INFO org.apache.hadoop.hbase.zookeeper.MetaNodeTracker: Detected completed assignment of META, notifying catalog tracker",
      "2011-04-12 15:00:45,870 INFO org.apache.hadoop.hbase.master.handler.ServerShutdownHandler: Reassigning 0 region(s) that C4C3.site,60020,1302590910433 was carrying (skipping 0 regions(s) that are already in transition)",
      "2011-04-12 15:00:45,870 INFO org.apache.hadoop.hbase.master.handler.ServerShutdownHandler: Finished processing of shutdown of C4C3.site,60020,1302590910433"
    ],
    "stack_traces": [
      "java.net.ConnectException: Call to C4C1/157.5.100.1:9000 failed on connection exception: java.net.ConnectException: Connection refused\nat org.apache.hadoop.ipc.Client.wrapException(Client.java:844)\nat org.apache.hadoop.ipc.Client.call(Client.java:820)\nat org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:221)\nat $Proxy5.getListing(Unknown Source)\nat sun.reflect.GeneratedMethodAccessor9.invoke(Unknown Source)\nat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\nat java.lang.reflect.Method.invoke(Method.java:597)\nat org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82)\nat org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59)\nat $Proxy5.getListing(Unknown Source)\nat org.apache.hadoop.hdfs.DFSClient.listPaths(DFSClient.java:614)\nat org.apache.hadoop.hdfs.DistributedFileSystem.listStatus(DistributedFileSystem.java:252)\nat org.apache.hadoop.hbase.master.LogCleaner.chore(LogCleaner.java:121)\nat org.apache.hadoop.hbase.Chore.run(Chore.java:66)\nat org.apache.hadoop.hbase.master.LogCleaner.run(LogCleaner.java:154)\nCaused by: java.net.ConnectException: Connection refused\nat sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)\nat sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)\nat org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)\nat org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)\nat org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:332)\nat org.apache.hadoop.ipc.Client$Connection.access$2000(Client.java:202)\nat org.apache.hadoop.ipc.Client.getConnection(Client.java:943)\nat org.apache.hadoop.ipc.Client.call(Client.java:788)\n... 13 more",
      "java.net.ConnectException: Call to C4C1/157.5.100.1:9000 failed on connection exception: java.net.ConnectException: Connection refused\nat org.apache.hadoop.ipc.Client.wrapException(Client.java:844)\nat org.apache.hadoop.ipc.Client.call(Client.java:820)\nat org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:221)\nat $Proxy5.getFileInfo(Unknown Source)\nat sun.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)\nat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\nat java.lang.reflect.Method.invoke(Method.java:597)\nat org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82)\nat org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59)\nat $Proxy5.getFileInfo(Unknown Source)\nat org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:623)\nat org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:461)\nat org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:690)\nat org.apache.hadoop.hbase.regionserver.wal.HLogSplitter.splitLog(HLogSplitter.java:177)\nat org.apache.hadoop.hbase.master.MasterFileSystem.splitLog(MasterFileSystem.java:196)\nat org.apache.hadoop.hbase.master.handler.ServerShutdownHandler.process(ServerShutdownHandler.java:95)\nat org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:151)\nat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\nat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\nat java.lang.Thread.run(Thread.java:662)\nCaused by: java.net.ConnectException: Connection refused\nat sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)\nat sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)\nat org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)\nat org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)\nat org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:332)\nat org.apache.hadoop.ipc.Client$Connection.access$2000(Client.java:202)\nat org.apache.hadoop.ipc.Client.getConnection(Client.java:943)\nat org.apache.hadoop.ipc.Client.call(Client.java:788)\n... 18 more"
    ]
  },
  {
    "file": "HBase-3989.docx",
    "title": "HBase-3989",
    "version": "hbase-0.90.3",
    "Description": "Error occured while RegionServer report to Master \"we are up\" should get master address again",
    "logs": [
      "2011-06-13 11:25:12,236 WARN org.apache.hadoop.hbase.regionserver.HRegionServer: error telling master we are up",
      "2011-06-13 11:25:15,231 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: Telling master at 157-5-111-22:20000 that we are up",
      "2011-06-13 11:25:15,232 WARN org.apache.hadoop.hbase.regionserver.HRegionServer: error telling master we are up",
      "2011-06-13 11:25:18,225 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: Telling master at 157-5-111-22:20000 that we are up"
    ],
    "stack_traces": [
      "java.net.ConnectException: Connection refused\nat sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)\nat sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)\nat org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:207)\nat org.apache.hadoop.net.NetUtils.connect(NetUtils.java:419)\nat org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)\nat org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)\nat org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)\nat org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)\nat $Proxy5.regionServerStartup(Unknown Source)\nat org.apache.hadoop.hbase.regionserver.HRegionServer.reportForDuty(HRegionServer.java:1511)\nat org.apache.hadoop.hbase.regionserver.HRegionServer.tryReportForDuty(HRegionServer.java:1479)\nat org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:571)\nat java.lang.Thread.run(Thread.java:662)",
      "java.net.ConnectException: Connection refused\nat sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)\nat sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)\nat org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:207)\nat org.apache.hadoop.net.NetUtils.connect(NetUtils.java:419)\nat org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)\nat org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)\nat org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)\nat org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)\nat $Proxy5.regionServerStartup(Unknown Source)\nat org.apache.hadoop.hbase.regionserver.HRegionServer.reportForDuty(HRegionServer.java:1511)\nat org.apache.hadoop.hbase.regionserver.HRegionServer.tryReportForDuty(HRegionServer.java:1479)\nat org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:571)\nat java.lang.Thread.run(Thread.java:662)"
    ]
  },
  {
    "file": "HBase-4124.docx",
    "title": "HBase-4124",
    "version": "hbase-0.90.4",
    "Description": "ZK restarted while a region is being assigned, new active HM re-assigns it but the RS warns 'already online on this server'.",
    "logs": [
      "2011-07-21 07:05:44,712 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: master:20000-0x1314ac5addb003e Async create of unassigned node for ffe0f4ab26eaa1796d2f99565ff24012 with OFFLINE state",
      "2011-07-21 07:05:46,661 DEBUG org.apache.hadoop.hbase.master.AssignmentManager$CreateUnassignedAsyncCallback: rs=test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012. state=OFFLINE, ts=1311231944712, server=158-1-91-105,20020,1311231874887",
      "2011-07-21 07:05:46,837 DEBUG org.apache.hadoop.hbase.master.AssignmentManager$ExistsUnassignedAsyncCallback: rs=test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012. state=OFFLINE, ts=1311231944712",
      "2011-07-21 07:05:55,734 INFO org.apache.zookeeper.ClientCnxn: Unable to read additional data from server sessionid 0x1314ac5addb003f, likely server has closed socket, closing socket connection and attempting reconnect",
      "2011-07-21 07:05:55,835 FATAL org.apache.hadoop.hbase.master.HMaster: Error deleting OPENED node in ZK for transition ZK node (2c5d23581fc848c89e5be0555086cf30)",
      "2011-07-21 07:07:00,902 INFO org.apache.hadoop.hbase.master.ServerManager: Registering server=158-1-91-105,20020,1311231874887, regionCount=971, userLoad=true",
      "2011-07-21 07:07:01,569 INFO org.apache.hadoop.hbase.master.ServerManager: Registering server=158-1-91-101,20020,1311231878544, regionCount=951, userLoad=true",
      "2011-07-21 07:07:36,903 INFO org.apache.hadoop.hbase.master.ServerManager: Registering server=158-1-91-103,20020,1311232056655, regionCount=0, userLoad=false",
      "2011-07-21 07:08:03,239 INFO org.apache.hadoop.hbase.master.metrics.MasterMetrics: Initialized",
      "2011-07-21 07:08:03,288 INFO org.apache.hadoop.hbase.master.ActiveMasterManager: Another master is the active master, 158-1-91-103:20000; waiting to become the next active master",
      "2011-07-21 07:04:45,482 INFO org.apache.hadoop.hbase.ipc.HBaseRpcMetrics: Initializing RPC Metrics with hostName=HMaster, port=20000",
      "2011-07-21 07:15:13,719 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: master:20000-0x1314ac5addb0042-0x1314ac5addb0042 Creating (or updating) unassigned node for ffe0f4ab26eaa1796d2f99565ff24012 with OFFLINE state",
      "2011-07-21 07:30:26,526 INFO org.apache.hadoop.hbase.master.AssignmentManager: Processing region test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012. in state M_ZK_REGION_OFFLINE",
      "2011-07-21 07:30:26,562 DEBUG org.apache.hadoop.hbase.master.handler.ClosedRegionHandler: Handling CLOSED event for ffe0f4ab26eaa1796d2f99565ff24012",
      "2011-07-21 07:30:26,562 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Forcing OFFLINE; was=test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012. state=OFFLINE, ts=1311232513720",
      "2011-07-21 07:30:26,562 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: master:20000-0x1314ac5addb0042-0x1314ac5addb0042 Creating (or updating) unassigned node for ffe0f4ab26eaa1796d2f99565ff24012 with OFFLINE state",
      "2011-07-21 07:30:26,577 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: No previous transition plan was found (or we are ignoring an existing plan) for test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012. so generated a random one; hri=test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012., src=, dest=158-1-91-105,20020,1311231874887; 3 (online=3, exclude=null) available servers",
      "2011-07-21 07:30:26,577 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Assigning region test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012. to 158-1-91-105,20020,1311231874887",
      "2011-07-21 07:30:26,603 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Handling transition=M_ZK_REGION_OFFLINE, server=158-1-91-103:20000, region=ffe0f4ab26eaa1796d2f99565ff24012",
      "2011-07-21 07:42:08,737 INFO org.apache.hadoop.hbase.master.AssignmentManager: Regions in transition timed out:  test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012. state=PENDING_OPEN, ts=1311233942949",
      "2011-07-21 07:42:08,737 INFO org.apache.hadoop.hbase.master.AssignmentManager: Region has been PENDING_OPEN for too long, reassigning region=test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012.",
      "2011-07-21 07:42:49,043 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Forcing OFFLINE; was=test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012. state=PENDING_OPEN, ts=1311234169040",
      "2011-07-21 07:42:49,043 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: No previous transition plan was found (or we are ignoring an existing plan) for test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012. so generated a random one; hri=test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012., src=, dest=158-1-91-105,20020,1311231874887; 3 (online=3, exclude=null) available servers",
      "2011-07-21 07:42:49,043 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Assigning region test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012. to 158-1-91-105,20020,1311231874887",
      "2011-07-21 07:46:25,809 INFO org.apache.hadoop.hbase.master.AssignmentManager: Regions in transition timed out:  test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012. state=PENDING_OPEN, ts=1311234201519",
      "2011-07-21 07:46:25,809 INFO org.apache.hadoop.hbase.master.AssignmentManager: Region has been PENDING_OPEN for too long, reassigning region=test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012.",
      "2011-07-21 07:46:31,872 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Forcing OFFLINE; was=test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012. state=PENDING_OPEN, ts=1311234391597",
      "2011-07-21 07:46:31,874 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: No previous transition plan was found (or we are ignoring an existing plan) for test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012. so generated a random one; hri=test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012., src=, dest=158-1-91-103,20020,1311232056655; 3 (online=3, exclude=null) available servers",
      "2011-07-21 07:46:31,874 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Assigning region test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012. to 158-1-91-103,20020,1311232056655",
      "2011-07-21 07:46:32,061 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Handling transition=RS_ZK_REGION_OPENED, server=158-1-91-103,20020,1311232056655, region=ffe0f4ab26eaa1796d2f99565ff24012",
      "2011-07-21 07:46:32,493 DEBUG org.apache.hadoop.hbase.master.handler.OpenedRegionHandler: Handling OPENED event for ffe0f4ab26eaa1796d2f99565ff24012; deleting unassigned node",
      "2011-07-21 07:46:32,493 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: master:20000-0x1314ac5addb0042-0x1314ac5addb0042 Deleting existing unassigned node for ffe0f4ab26eaa1796d2f99565ff24012 that is in expected state RS_ZK_REGION_OPENED",
      "2011-07-21 07:46:32,693 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: master:20000-0x1314ac5addb0042-0x1314ac5addb0042 Successfully deleted unassigned node for region ffe0f4ab26eaa1796d2f99565ff24012 in expected state RS_ZK_REGION_OPENED",
      "2011-07-21 07:46:32,717 DEBUG org.apache.hadoop.hbase.master.handler.OpenedRegionHandler: Opened region test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012. on 158-1-91-103,20020,1311232056655",
      "2011-07-21 07:05:48,091 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: Received request to open region: test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012.",
      "2011-07-21 07:08:44,043 DEBUG org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler: Processing open of test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012.",
      "2011-07-21 07:08:44,043 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:20020-0x2314ac6763f0027 Attempting to transition node ffe0f4ab26eaa1796d2f99565ff24012 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING",
      "2011-07-21 07:08:44,106 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:20020-0x2314ac6763f0027 Successfully transitioned node ffe0f4ab26eaa1796d2f99565ff24012 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING",
      "2011-07-21 07:08:44,106 DEBUG org.apache.hadoop.hbase.regionserver.HRegion: Opening region: REGION => {NAME => 'test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012.', STARTKEY => '282139', ENDKEY => '282142', ENCODED => ffe0f4ab26eaa1796d2f99565ff24012, TABLE => {{NAME => 'test1', FAMILIES => [{NAME => 'value', BLOOMFILTER => 'NONE', REPLICATION_SCOPE => '0', VERSIONS => '3', COMPRESSION => 'GZ', TTL => '2592000', BLOCKSIZE => '65536', IN_MEMORY => 'false', BLOCKCACHE => 'true'}]}}",
      "2011-07-21 07:08:44,106 DEBUG org.apache.hadoop.hbase.regionserver.HRegion: Instantiated test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012.",
      "2011-07-21 07:08:44,144 INFO org.apache.hadoop.hbase.regionserver.HRegion: Onlined test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012.; next sequenceid=1",
      "2011-07-21 07:08:44,144 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:20020-0x2314ac6763f0027 Attempting to transition node ffe0f4ab26eaa1796d2f99565ff24012 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING",
      "2011-07-21 07:08:44,202 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:20020-0x2314ac6763f0027 Successfully transitioned node ffe0f4ab26eaa1796d2f99565ff24012 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING",
      "2011-07-21 07:08:44,232 INFO org.apache.hadoop.hbase.catalog.MetaEditor: Updated row test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012. in region .META.,,1 with server=158-1-91-105:20020, startcode=1311231874887",
      "2011-07-21 07:08:44,232 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:20020-0x2314ac6763f0027 Attempting to transition node ffe0f4ab26eaa1796d2f99565ff24012 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED",
      "2011-07-21 07:08:44,291 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:20020-0x2314ac6763f0027 Successfully transitioned node ffe0f4ab26eaa1796d2f99565ff24012 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED",
      "2011-07-21 07:08:44,291 DEBUG org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler: Opened test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012.  It shows that RS105 open the region4012 successfully.",
      "2011-07-21 07:30:26,579 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: Received request to open region: test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012.",
      "2011-07-21 07:30:26,579 DEBUG org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler: Processing open of test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012.",
      "2011-07-21 07:30:26,579 WARN org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler: Attempted open of test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012. but already online on this server         This suggests that the assignment from HM101 is successful in RS105 (since region4012 is in RS105’s onlineRegion list), although the state in zk for this assignment is offline.",
      "2011-07-21 07:42:49,046 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: Received request to open region: test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012.",
      "2011-07-21 07:42:49,046 DEBUG org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler: Processing open of test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012.",
      "2011-07-21 07:42:49,046 WARN org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler: Attempted open of test1,282139,1311216321932.ffe0f4ab26eaa1796d2f99565ff24012. but already online on this server"
    ],
    "stack_traces": [
      "at org.apache.zookeeper.KeeperException.create(KeeperException.java:90)\nat org.apache.zookeeper.KeeperException.create(KeeperException.java:42)\nat org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:738)\nat org.apache.hadoop.hbase.zookeeper.ZKUtil.deleteNode(ZKUtil.java:962)"
    ]
  },
  {
    "file": "HBase-4400.docx",
    "title": "HBase-4400",
    "version": "hbase-0.90.4",
    "Description": ".META. getting stuck if RS hosting it is dead and znode state is in RS_ZK_REGION_OPENED",
    "logs": [
      "2011-09-14 16:43:51,949 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Handling transition=RS_ZK_REGION_OPENING, server=linux76,60020,1315998828523, region=70236052/-ROOT-",
      "2011-09-14 16:43:51,968 INFO org.apache.hadoop.hbase.master.HMaster: -ROOT- assigned=1, rit=false, location=linux76:60020",
      "2011-09-14 16:43:51,970 INFO org.apache.hadoop.hbase.master.AssignmentManager: Processing region .META.,,1.1028785192 in state RS_ZK_REGION_OPENED",
      "2011-09-14 16:43:51,970 INFO org.apache.hadoop.hbase.master.AssignmentManager: Failed to find linux146,60020,1315998414623 in list of online servers; skipping registration of open of .META.,,1.1028785192",
      "2011-09-14 16:43:51,971 INFO org.apache.hadoop.hbase.master.AssignmentManager: Waiting on 1028785192/.META.",
      "2011-09-14 16:43:51,983 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Handling transition=RS_ZK_REGION_OPENED, server=linux76,60020,1315998828523, region=70236052/-ROOT-",
      "2011-09-14 16:43:51,986 DEBUG org.apache.hadoop.hbase.master.handler.OpenedRegionHandler: Handling OPENED event for 70236052; deleting unassigned node",
      "2011-09-14 16:43:51,986 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: master:60000-0x13267854032001d Deleting existing unassigned node for 70236052 that is in expected state RS_ZK_REGION_OPENED",
      "2011-09-14 16:43:51,998 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: master:60000-0x13267854032001d Successfully deleted unassigned node for region 70236052 in expected state RS_ZK_REGION_OPENED",
      "2011-09-14 16:43:51,999 DEBUG org.apache.hadoop.hbase.master.handler.OpenedRegionHandler: Opened region -ROOT-,,0.70236052 on linux76,60020,1315998828523",
      "2011-09-14 16:44:00,945 INFO org.apache.hadoop.hbase.master.ServerManager: Registering server=linux146,60020,1315998839724, regionCount=0, userLoad=false",
      "2011-09-14 16:46:20,003 INFO org.apache.hadoop.hbase.master.AssignmentManager: Regions in transition timed out: .META.,,1.1028785192 state=OPEN, ts=0",
      "2011-09-14 16:46:20,004 ERROR org.apache.hadoop.hbase.master.AssignmentManager: Region has been OPEN for too long, we don't know where region was opened so can't do anything"
    ],
    "stack_traces": []
  },
  {
    "file": "HBase-4470.docx",
    "title": "HBase-4470",
    "version": "hbase-0.92.1",
    "Description": "ServerNotRunningException coming out of assignRootAndMeta kills the Maste",
    "logs": [
      "2011-09-23 04:47:44,859 WARN org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation: RemoteException connecting to RS"
    ],
    "stack_traces": [
      "org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hbase.ipc.ServerNotRunningException: Server is not running yet\nat org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1038)\nat org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:771)\nat org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)\nat $Proxy6.getProtocolVersion(Unknown Source)\nat org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)\nat org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)\nat org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)\nat org.apache.hadoop.hbase.ipc.HBaseRPC.waitForProxy(HBaseRPC.java:349)\nat org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:969)\nat org.apache.hadoop.hbase.catalog.CatalogTracker.getCachedConnection(CatalogTracker.java:388)\nat org.apache.hadoop.hbase.catalog.CatalogTracker.getMetaServerConnection(CatalogTracker.java:287)\nat org.apache.hadoop.hbase.catalog.CatalogTracker.verifyMetaRegionLocation(CatalogTracker.java:484)\nat org.apache.hadoop.hbase.master.HMaster.assignRootAndMeta(HMaster.java:441)\nat org.apache.hadoop.hbase.master.HMaster.finishInitialization(HMaster.java:388)\nat org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:282)"
    ]
  },
  {
    "file": "HBase-4729.docx",
    "title": "HBase-4729",
    "version": "hbase-0.92.0",
    "Description": "Clash between region unassign and splitting kills the master",
    "logs": [
      "2011-11-02 17:06:44,428 FATAL org.apache.hadoop.hbase.master.HMaster: Unexpected ZK exception creating node CLOSING",
      "2011-11-02 17:06:40,704 INFO org.apache.hadoop.hbase.regionserver.SplitTransaction: Starting split of region TestTable,0012469153,1320253135043.f7e1783e65ea8d621a4bc96ad310f101.",
      "2011-11-02 17:06:40,704 DEBUG org.apache.hadoop.hbase.regionserver.SplitTransaction: regionserver:62023-0x132f043bbde0710 Creating ephemeral node for f7e1783e65ea8d621a4bc96ad310f101 in SPLITTING state",
      "2011-11-02 17:06:40,751 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:62023-0x132f043bbde0710 Attempting to transition node f7e1783e65ea8d621a4bc96ad310f101 from RS_ZK_REGION_SPLITTING to RS_ZK_REGION_SPLITTING",
      "2011-11-02 17:06:44,061 INFO org.apache.hadoop.hbase.regionserver.SplitTransaction: Still waiting on the master to process the split for f7e1783e65ea8d621a4bc96ad310f101"
    ],
    "stack_traces": [
      "at org.apache.zookeeper.KeeperException.create(KeeperException.java:110)\nat org.apache.zookeeper.KeeperException.create(KeeperException.java:42)\nat org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:637)\nat org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.createNonSequential(RecoverableZooKeeper.java:459)\nat org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.create(RecoverableZooKeeper.java:441)\nat org.apache.hadoop.hbase.zookeeper.ZKUtil.createAndWatch(ZKUtil.java:769)\nat org.apache.hadoop.hbase.zookeeper.ZKAssign.createNodeClosing(ZKAssign.java:568)\nat org.apache.hadoop.hbase.master.AssignmentManager.unassign(AssignmentManager.java:1722)\nat org.apache.hadoop.hbase.master.AssignmentManager.unassign(AssignmentManager.java:1661)\nat org.apache.hadoop.hbase.master.BulkReOpen$1.run(BulkReOpen.java:69)\nat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\nat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\nat java.lang.Thread.run(Thread.java:662)"
    ]
  },
  {
    "file": "HBase-4792.docx",
    "title": "HBase-4792",
    "version": "hbase-0.92.0",
    "Description": "SplitRegionHandler doesn't care if it deletes the znode or not, leaves the parent region stuck offline",
    "logs": [
      "2011-11-15 22:28:57,900 DEBUG org.apache.hadoop.hbase.master.handler.SplitRegionHandler: Handling SPLIT event for e5be6551c8584a6a1065466e520faf4e; deleting node",
      "2011-11-15 22:28:57,900 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: master:62003-0x132f043bbde08c1 Deleting existing unassigned node for e5be6551c8584a6a1065466e520faf4e that is in expected state RS_ZK_REGION_SPLIT",
      "2011-11-15 22:28:57,975 WARN org.apache.hadoop.hbase.zookeeper.ZKAssign: master:62003-0x132f043bbde08c1 Attempting to delete unassigned node in RS_ZK_REGION_SPLIT state but after verifying state, we got a version mismatch",
      "2011-11-15 22:28:57,975 INFO org.apache.hadoop.hbase.master.handler.SplitRegionHandler: Handled SPLIT report); parent=TestTable,0001355346,1321396080924.e5be6551c8584a6a1065466e520faf4e. daughter a=TestTable,0001355346,1321396132414.df9b549eb594a1f8728608a2a431224a. daughter b=TestTable,0001368082,1321396132414.de861596db4337dc341138f26b9c8bc2.",
      "2011-11-15 22:28:58,052 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Handling transition=RS_ZK_REGION_SPLIT, server=sv4r28s44,62023,1321395865619, region=e5be6551c8584a6a1065466e520faf4e",
      "2011-11-15 22:28:58,052 WARN org.apache.hadoop.hbase.master.AssignmentManager: Region e5be6551c8584a6a1065466e520faf4e not found on server sv4r28s44,62023,1321395865619; failed processing",
      "2011-11-15 22:28:58,052 WARN org.apache.hadoop.hbase.master.AssignmentManager: Received SPLIT for region e5be6551c8584a6a1065466e520faf4e from server sv4r28s44,62023,1321395865619 but it doesn't exist anymore, probably already processed its split",
      "2011-11-15 22:28:57,661 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:62023-0x132f043bbde08d3 Attempting to transition node e5be6551c8584a6a1065466e520faf4e from RS_ZK_REGION_SPLITTING to RS_ZK_REGION_SPLIT",
      "2011-11-15 22:28:57,775 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:62023-0x132f043bbde08d3 Successfully transitioned node e5be6551c8584a6a1065466e520faf4e from RS_ZK_REGION_SPLITTING to RS_ZK_REGION_SPLIT",
      "2011-11-15 22:28:57,775 INFO org.apache.hadoop.hbase.regionserver.SplitTransaction: Still waiting on the master to process the split for e5be6551c8584a6a1065466e520faf4e",
      "2011-11-15 22:28:57,876 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:62023-0x132f043bbde08d3 Attempting to transition node e5be6551c8584a6a1065466e520faf4e from RS_ZK_REGION_SPLIT to RS_ZK_REGION_SPLIT",
      "2011-11-15 22:28:57,967 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: regionserver:62023-0x132f043bbde08d3 Successfully transitioned node e5be6551c8584a6a1065466e520faf4e from RS_ZK_REGION_SPLIT to RS_ZK_REGION_SPLIT"
    ],
    "stack_traces": []
  },
  {
    "file": "HBase-5063.docx",
    "title": "HBase-5063",
    "version": "hbase-0.90.6",
    "Description": "RegionServers fail to report to backup HMaster after primary goes down",
    "logs": [
      "11/12/17 18:50:24 WARN regionserver.HRegionServer: Unable to connect to master. Retrying. Error was:"
    ],
    "stack_traces": [
      "java.net.ConnectException: Connection refused\n\t\tat sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)\n\t\tat sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)\n\t\tat org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)\n\t\tat org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)\n\t\tat org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupConnection(HBaseClient.java:328)\n\t\tat org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:362)\n\t\tat org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:1024)\n\t\tat org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:876)\n\t\tat org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:150)\n\t\tat $Proxy8.getProtocolVersion(Unknown Source)\n\t\tat org.apache.hadoop.hbase.ipc.WritableRpcEngine.getProxy(WritableRpcEngine.java:183)\n\t\tat org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:303)\n\t\tat org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:280)\n\t\tat org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:332)\n\t\tat org.apache.hadoop.hbase.ipc.HBaseRPC.waitForProxy(HBaseRPC.java:236)\n\t\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getMaster(HRegionServer.java:1616)\n\t\tat org.apache.hadoop.hbase.regionserver.HRegionServer.tryRegionServerReport(HRegionServer.java:787)\n\t\tat org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:674)\n\t\tat java.lang.Thread.run(Thread.java:619)"
    ]
  },
  {
    "file": "HBase-5883.docx",
    "title": "HBase-5883",
    "version": "hbase-0.94.0",
    "Description": "Backup master is going down due to connection refused exception",
    "logs": [
      "2012-04-09 10:42:24,270 INFO org.apache.hadoop.hbase.master.SplitLogManager: finished splitting (more than or equal to) 861248320 bytes in 4 log files in [hdfs://192.168.47.205:9000/hbase/.logs/HOST-192-168-47-202,60020,1333715537172-splitting] in 26374ms",
      "2012-04-09 10:42:24,316 FATAL org.apache.hadoop.hbase.master.HMaster: Master server abort: loaded coprocessors are: []",
      "2012-04-09 10:42:24,333 FATAL org.apache.hadoop.hbase.master.HMaster: Unhandled exception. Starting shutdown.",
      "2012-04-09 10:42:24,336 INFO org.apache.hadoop.hbase.master.HMaster: Aborting",
      "2012-04-09 10:42:24,336 DEBUG org.apache.hadoop.hbase.master.HMaster: Stopping service threads"
    ],
    "stack_traces": [
      "java.io.IOException: java.net.ConnectException: Connection refused\n\t\tat org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:375)\n\t\tat org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:1045)\n\t\tat org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:897)\n\t\tat org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:150)\n\t\tat $Proxy13.getProtocolVersion(Unknown Source)\n\t\tat org.apache.hadoop.hbase.ipc.WritableRpcEngine.getProxy(WritableRpcEngine.java:183)\n\t\tat org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:303)\n\t\tat org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:280)\n\t\tat org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:332)\n\t\tat org.apache.hadoop.hbase.ipc.HBaseRPC.waitForProxy(HBaseRPC.java:236)\n\t\tat org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:1276)\n\t\tat org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:1233)\n\t\tat org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:1220)\n\t\tat org.apache.hadoop.hbase.catalog.CatalogTracker.getCachedConnection(CatalogTracker.java:569)\n\t\tat org.apache.hadoop.hbase.catalog.CatalogTracker.getRootServerConnection(CatalogTracker.java:369)\n\t\tat org.apache.hadoop.hbase.catalog.CatalogTracker.waitForRootServerConnection(CatalogTracker.java:353)\n\t\tat org.apache.hadoop.hbase.catalog.CatalogTracker.verifyRootRegionLocation(CatalogTracker.java:660)\n\t\tat org.apache.hadoop.hbase.master.HMaster.assignRootAndMeta(HMaster.java:616)\n\t\tat org.apache.hadoop.hbase.master.HMaster.finishInitialization(HMaster.java:540)\n\t\tat org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:363)\n\t\tat java.lang.Thread.run(Thread.java:662)\nCaused by: java.net.ConnectException: Connection refused\n\t\tat sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)\n\t\tat sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:567)\n\t\tat org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)\n\t\tat org.apache.hadoop.net.NetUtils.connect(NetUtils.java:488)\n\t\tat org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupConnection(HBaseClient.java:328)\n\t\tat org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:362)\n\t\t... 20 more"
    ]
  },
  {
    "file": "HBase-5927.docx",
    "title": "HBase-5927",
    "version": "hbase-0.94.0",
    "Description": "SSH and DisableTableHandler happening together does not clear the znode of the region and RIT map.",
    "logs": [
      "2012-05-08 22:04:03,038 INFO [MASTER_SERVER_OPERATIONS-Jieshan.china.huawei.com,57110,1336485831968-0] handler.ServerShutdownHandler(174): Splitting logs for Jieshan.china.huawei.com,57163,1336485834464",
      "2012-05-08 22:05:23,909 INFO [Jieshan.china.huawei.com,57110,1336485831968-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-1] master.AssignmentManager(2066): Server jieshan.china.huawei.com,57138,1336485834409 returned java.net.ConnectException: Connection refused: no further information for fc6c3b7c6445127b63f617445aa5dfaf",
      "2012-05-08 22:07:58,122 INFO [Jieshan.china.huawei.com,57110,1336485831968.timeoutMonitor] master.AssignmentManager$TimeoutMonitor(2925): Region has been PENDING_CLOSE for too long, running forced unassign again on region=testHBase5937,000,1336485845463.fc6c3b7c6445127b63f617445aa5dfaf.",
      "2012-05-08 22:07:58,122 DEBUG [pool-8-thread-17] master.AssignmentManager(1942): Attempted to unassign region testHBase5937,000,1336485845463.fc6c3b7c6445127b63f617445aa5dfaf. but it is not currently assigned anywhere",
      "2012-05-08 22:08:08,121 INFO [Jieshan.china.huawei.com,57110,1336485831968.timeoutMonitor] master.AssignmentManager$TimeoutMonitor(2893): Regions in transition timed out: testHBase5937,000,1336485845463.fc6c3b7c6445127b63f617445aa5dfaf. state=PENDING_CLOSE, ts=1336485917495, server=null",
      "2012-05-08 22:08:08,121 DEBUG [pool-8-thread-13] master.AssignmentManager(1942): Attempted to unassign region testHBase5937,000,1336485845463.fc6c3b7c6445127b63f617445aa5dfaf. but it is not currently assigned anywhere",
      "2012-05-08 22:08:18,122 INFO [Jieshan.china.huawei.com,57110,1336485831968.timeoutMonitor] master.AssignmentManager$TimeoutMonitor(2925): Region has been PENDING_CLOSE for too long, running forced unassign again on region=testHBase5937,000,1336485845463.fc6c3b7c6445127b63f617445aa5dfaf.",
      "2012-05-08 22:08:18,125 DEBUG [pool-8-thread-6] master.AssignmentManager(1942): Attempted to unassign region testHBase5937,000,1336485845463.fc6c3b7c6445127b63f617445aa5dfaf. but it is not currently assigned anywhere"
    ],
    "stack_traces": []
  },
  {
    "file": "HBase-6088.docx",
    "title": "HBase-6088",
    "version": "hbase-0.94.0",
    "Description": "Region splitting not happened for long time due to ZK exception while creating RS_ZK_SPLITTING node",
    "logs": [
      "2012-05-24 01:45:41,363 INFO org.apache.zookeeper.ClientCnxn: Client session timed out, have not heard from server in 26668ms for sessionid 0x1377a75f41d0012, closing socket connection and attempting reconnect",
      "2012-05-24 01:45:41,464 WARN org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper: Possibly transient ZooKeeper exception: org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss for /hbase/unassigned/bd1079bf948c672e493432020dc0e144",
      "2012-05-24 01:45:43,300 DEBUG org.apache.hadoop.hbase.regionserver.wal.HLog: cleanupCurrentWriter waiting for transactions to get synced total 189377 synced till here 189365",
      "2012-05-24 01:45:48,474 INFO org.apache.hadoop.hbase.regionserver.SplitRequest: Running rollback/cleanup of failed split of ufdr,011365398471659,1337823505339.bd1079bf948c672e493432020dc0e144.; Failed setting SPLITTING znode on ufdr,011365398471659,1337823505339.bd1079bf948c672e493432020dc0e144.",
      "2012-05-24 01:45:48,476 INFO org.apache.hadoop.hbase.regionserver.SplitRequest: Successful rollback of failed split of ufdr,011365398471659,1337823505339.bd1079bf948c672e493432020dc0e144.",
      "2012-05-24 01:47:28,141 ERROR org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper: Node /hbase/unassigned/bd1079bf948c672e493432020dc0e144 already exists and this is not a retry",
      "2012-05-24 01:47:28,142 INFO org.apache.hadoop.hbase.regionserver.SplitRequest: Running rollback/cleanup of failed split of ufdr,011365398471659,1337823505339.bd1079bf948c672e493432020dc0e144.; Failed create of ephemeral /hbase/unassigned/bd1079bf948c672e493432020dc0e144"
    ],
    "stack_traces": [
      "at org.apache.hadoop.hbase.regionserver.SplitTransaction.createDaughters(SplitTransaction.java:242)\nat org.apache.hadoop.hbase.regionserver.SplitTransaction.execute(SplitTransaction.java:450)\nat org.apache.hadoop.hbase.regionserver.SplitRequest.run(SplitRequest.java:67)\nat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\nat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\nat java.lang.Thread.run(Thread.java:662)\nCaused by: org.apache.zookeeper.KeeperException$BadVersionException: KeeperErrorCode = BadVersion for /hbase/unassigned/bd1079bf948c672e493432020dc0e144\nat org.apache.zookeeper.KeeperException.create(KeeperException.java:115)\nat org.apache.zookeeper.KeeperException.create(KeeperException.java:51)\nat org.apache.zookeeper.ZooKeeper.setData(ZooKeeper.java:1246)\nat org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.setData(RecoverableZooKeeper.java:321)\nat org.apache.hadoop.hbase.zookeeper.ZKUtil.setData(ZKUtil.java:659)\nat org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:811)\nat org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:747)\nat org.apache.hadoop.hbase.regionserver.SplitTransaction.transitionNodeSplitting(SplitTransaction.java:919)\nat org.apache.hadoop.hbase.regionserver.SplitTransaction.createNodeSplitting(SplitTransaction.java:869)\nat org.apache.hadoop.hbase.regionserver.SplitTransaction.createDaughters(SplitTransaction.java:239)\n... 5 more",
      "at org.apache.hadoop.hbase.regionserver.SplitTransaction.createNodeSplitting(SplitTransaction.java:865)\nat org.apache.hadoop.hbase.regionserver.SplitTransaction.createDaughters(SplitTransaction.java:239)\nat org.apache.hadoop.hbase.regionserver.SplitTransaction.execute(SplitTransaction.java:450)\nat org.apache.hadoop.hbase.regionserver.SplitRequest.run(SplitRequest.java:67)"
    ]
  },
  {
    "file": "HBase-6299.docx",
    "title": "HBase-6299",
    "version": "hbase-0.94.1",
    "Description": "RS starting region open while failing ack to HMaster.sendRegionOpen() causes inconsistency in HMaster's region state and a series of successive problems",
    "logs": [
      "2012-06-29 07:03:38,870 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Using pre-existing plan for region CDR_STATS_TRAFFIC,13184390567|20120508|17||2|3|913,1337256975556.b713fd655fa02395496c5a6e39ddf568.; plan=hri=CDR_STATS_TRAFFIC,13184390567|20120508|17||2|3|913,1337256975556.b713fd655fa02395496c5a6e39ddf568., src=swbss-hadoop-004,60020,1340890123243, dest=swbss-hadoop-006,60020,1340890678078",
      "2012-06-29 07:03:38,870 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Assigning region CDR_STATS_TRAFFIC,13184390567|20120508|17||2|3|913,1337256975556.b713fd655fa02395496c5a6e39ddf568. to swbss-hadoop-006,60020,1340890678078",
      "2012-06-29 07:03:38,870 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Handling transition=M_ZK_REGION_OFFLINE, server=swbss-hadoop-002:60000, region=b713fd655fa02395496c5a6e39ddf568",
      "2012-06-29 07:06:28,882 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Handling transition=RS_ZK_REGION_OPENING, server=swbss-hadoop-006,60020,1340890678078, region=b713fd655fa02395496c5a6e39ddf568",
      "2012-06-29 07:06:32,291 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Handling transition=RS_ZK_REGION_OPENING, server=swbss-hadoop-006,60020,1340890678078, region=b713fd655fa02395496c5a6e39ddf568",
      "2012-06-29 07:06:32,299 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Handling transition=RS_ZK_REGION_OPENED, server=swbss-hadoop-006,60020,1340890678078, region=b713fd655fa02395496c5a6e39ddf568",
      "2012-06-29 07:06:32,299 DEBUG org.apache.hadoop.hbase.master.handler.OpenedRegionHandler: Handling OPENED event for CDR_STATS_TRAFFIC,13184390567|20120508|17||2|3|913,1337256975556.b713fd655fa02395496c5a6e39ddf568. from serverName=swbss-hadoop-006,60020,1340890678078, load=(requests=518945, regions=575, usedHeap=15282, maxHeap=31301); deleting unassigned node",
      "2012-06-29 07:06:32,299 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: master:60000-0x2377fee2ae80007 Deleting existing unassigned node for b713fd655fa02395496c5a6e39ddf568 that is in expected state RS_ZK_REGION_OPENED",
      "2012-06-29 07:06:32,301 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: master:60000-0x2377fee2ae80007 Successfully deleted unassigned node for region b713fd655fa02395496c5a6e39ddf568 in expected state RS_ZK_REGION_OPENED",
      "2012-06-29 07:06:32,301 DEBUG org.apache.hadoop.hbase.master.handler.OpenedRegionHandler: The master has opened the region CDR_STATS_TRAFFIC,13184390567|20120508|17||2|3|913,1337256975556.b713fd655fa02395496c5a6e39ddf568. that was online on serverName=swbss-hadoop-006,60020,1340890678078, load=(requests=518945, regions=575, usedHeap=15282, maxHeap=31301)",
      "2012-06-29 07:07:41,140 WARN org.apache.hadoop.hbase.master.AssignmentManager: Failed assignment of CDR_STATS_TRAFFIC,13184390567|20120508|17||2|3|913,1337256975556.b713fd655fa02395496c5a6e39ddf568. to serverName=swbss-hadoop-006,60020,1340890678078, load=(requests=0, regions=575, usedHeap=0, maxHeap=0), trying to assign elsewhere instead; retry=0",
      "2012-06-29 07:07:41,142 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: No previous transition plan was found (or we are ignoring an existing plan) for CDR_STATS_TRAFFIC,13184390567|20120508|17||2|3|913,1337256975556.b713fd655fa02395496c5a6e39ddf568. so generated a random one; hri=CDR_STATS_TRAFFIC,13184390567|20120508|17||2|3|913,1337256975556.b713fd655fa02395496c5a6e39ddf568., src=, dest=swbss-hadoop-164,60020,1340888346294; 15 (online=15, exclude=serverName=swbss-hadoop-006,60020,1340890678078, load=(requests=0, regions=575, usedHeap=0, maxHeap=0)) available servers",
      "2012-06-29 07:07:41,142 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: master:60000-0x2377fee2ae80007 Creating (or updating) unassigned node for b713fd655fa02395496c5a6e39ddf568 with OFFLINE state",
      "2012-06-29 07:07:41,145 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Using pre-existing plan for region CDR_STATS_TRAFFIC,13184390567|20120508|17||2|3|913,1337256975556.b713fd655fa02395496c5a6e39ddf568.; plan=hri=CDR_STATS_TRAFFIC,13184390567|20120508|17||2|3|913,1337256975556.b713fd655fa02395496c5a6e39ddf568., src=, dest=swbss-hadoop-164,60020,1340888346294",
      "2012-06-29 07:07:41,145 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Assigning region CDR_STATS_TRAFFIC,13184390567|20120508|17||2|3|913,1337256975556.b713fd655fa02395496c5a6e39ddf568. to swbss-hadoop-164,60020,1340888346294",
      "2012-06-29 07:07:41,149 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Handling transition=RS_ZK_REGION_OPENING, server=swbss-hadoop-164,60020,1340888346294, region=b713fd655fa02395496c5a6e39ddf568",
      "2012-06-29 07:07:41,150 WARN org.apache.hadoop.hbase.master.AssignmentManager: Received OPENING for region b713fd655fa02395496c5a6e39ddf568 from server swbss-hadoop-164,60020,1340888346294 but region was in the state null and not in expected PENDING_OPEN or OPENING states",
      "2012-06-29 07:07:41,296 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Handling transition=RS_ZK_REGION_OPENING, server=swbss-hadoop-164,60020,1340888346294, region=b713fd655fa02395496c5a6e39ddf568",
      "2012-06-29 07:07:41,296 WARN org.apache.hadoop.hbase.master.AssignmentManager: Received OPENING for region b713fd655fa02395496c5a6e39ddf568 from server swbss-hadoop-164,60020,1340888346294 but region was in the state null and not in expected PENDING_OPEN or OPENING states",
      "2012-06-29 07:07:41,302 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Handling transition=RS_ZK_REGION_OPENED, server=swbss-hadoop-164,60020,1340888346294, region=b713fd655fa02395496c5a6e39ddf568",
      "2012-06-29 07:07:41,302 WARN org.apache.hadoop.hbase.master.AssignmentManager: Received OPENED for region b713fd655fa02395496c5a6e39ddf568 from server swbss-hadoop-164,60020,1340888346294 but region was in the state null and not in expected PENDING_OPEN or OPENING states",
      "2012-06-29 07:08:38,872 INFO org.apache.hadoop.hbase.master.HMaster: balance hri=CDR_STATS_TRAFFIC,13184390567|20120508|17||2|3|913,1337256975556.b713fd655fa02395496c5a6e39ddf568., src=swbss-hadoop-006,60020,1340890678078, dest=swbss-hadoop-008,60020,1340891085175",
      "2012-06-29 07:08:38,872 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Starting unassignment of region CDR_STATS_TRAFFIC,13184390567|20120508|17||2|3|913,1337256975556.b713fd655fa02395496c5a6e39ddf568. (offlining)",
      "2012-06-29 07:08:47,875 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Sent CLOSE to serverName=swbss-hadoop-006,60020,1340890678078, load=(requests=0, regions=0, usedHeap=0, maxHeap=0) for region CDR_STATS_TRAFFIC,13184390567|20120508|17||2|3|913,1337256975556.b713fd655fa02395496c5a6e39ddf568.",
      "2012-06-29 08:04:37,681 INFO org.apache.hadoop.hbase.master.AssignmentManager: Regions in transition timed out: CDR_STATS_TRAFFIC,13184390567|20120508|17||2|3|913,1337256975556.b713fd655fa02395496c5a6e39ddf568. state=PENDING_CLOSE, ts=1340926468331, server=null",
      "2012-06-29 08:04:37,681 INFO org.apache.hadoop.hbase.master.AssignmentManager: Region has been PENDING_CLOSE for too long, running forced unassign again on region=CDR_STATS_TRAFFIC,13184390567|20120508|17||2|3|913,1337256975556.b713fd655fa02395496c5a6e39ddf568.",
      "2012-06-29 08:04:47,681 INFO org.apache.hadoop.hbase.master.AssignmentManager: Regions in transition timed out: CDR_STATS_TRAFFIC,13184390567|20120508|17||2|3|913,1337256975556.b713fd655fa02395496c5a6e39ddf568. state=PENDING_CLOSE, ts=1340926468331, server=null",
      "2012-06-29 08:04:47,682 INFO org.apache.hadoop.hbase.master.AssignmentManager: Region has been PENDING_CLOSE for too long, running forced unassign again on region=CDR_STATS_TRAFFIC,13184390567|20120508|17||2|3|913,1337256975556.b713fd655fa02395496c5a6e39ddf568."
    ],
    "stack_traces": [
      "java.net.SocketTimeoutException: Call to /172.16.0.6:60020 failed on socket timeout exception: java.net.SocketTimeoutException: 120000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=/172.16.0.2:53765 remote=/172.16.0.6:60020]\n\t\tat org.apache.hadoop.hbase.ipc.HBaseClient.wrapException(HBaseClient.java:805)\n\t\tat org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:778)\n\t\tat org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:283)\n\t\tat $Proxy8.openRegion(Unknown Source)\n\t\tat org.apache.hadoop.hbase.master.ServerManager.sendRegionOpen(ServerManager.java:573)\n\t\tat org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1127)\n\t\tat org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:912)\n\t\tat org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:892)\n\t\tat org.apache.hadoop.hbase.master.handler.ClosedRegionHandler.process(ClosedRegionHandler.java:92)\n\t\tat org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:162)\n\t\tat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\n\t\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\n\t\tat java.lang.Thread.run(Thread.java:662)\nCaused by: java.net.SocketTimeoutException: 120000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=/172.16.0.2:53765 remote=/172.16.0.6:60020]\n\t\tat org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:164)\n\t\tat org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155)\n\t\tat org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128)\n\t\tat java.io.FilterInputStream.read(FilterInputStream.java:116)\n\t\tat org.apache.hadoop.hbase.ipc.HBaseClient$Connection$PingInputStream.read(HBaseClient.java:301)\n\t\tat java.io.BufferedInputStream.fill(BufferedInputStream.java:218)\n\t\tat java.io.BufferedInputStream.read(BufferedInputStream.java:237)\n\t\tat java.io.DataInputStream.readInt(DataInputStream.java:370)\n\t\tat org.apache.hadoop.hbase.ipc.HBaseClient$Connection.receiveResponse(HBaseClient.java:541)\n\t\tat org.apache.hadoop.hbase.ipc.HBaseClient$Connection.run(HBaseClient.java:479)"
    ]
  },
  {
    "file": "HBase-7504.docx",
    "title": "HBase-7504",
    "version": "hbase-0.94.3",
    "Description": "-ROOT- may be offline forever after FullGC of RS (Bug; Major)",
    "logs": [
      "2012-10-31 19:51:39,043 DEBUG org.apache.hadoop.hbase.master.ServerManager: Added=dw88.kgb.sqa.cm4,60020,1351671478752 to dead servers, submitted shutdown handler to be executed, root=true, meta=false",
      "2012-10-31 19:51:39,045 INFO org.apache.hadoop.hbase.master.handler.ServerShutdownHandler: Splitting logs for dw88.kgb.sqa.cm4,60020,1351671478752",
      "2012-10-31 19:51:50,113 INFO org.apache.hadoop.hbase.master.handler.ServerShutdownHandler: Server dw88.kgb.sqa.cm4,60020,1351671478752 was carrying ROOT. Trying to assign.",
      "2012-10-31 19:52:15,939 DEBUG org.apache.hadoop.hbase.master.ServerManager: Server REPORT rejected; currently processing dw88.kgb.sqa.cm4,60020,1351671478752 as dead server",
      "2012-10-31 19:52:15,945 INFO org.apache.hadoop.hbase.master.handler.ServerShutdownHandler: Skipping log splitting for dw88.kgb.sqa.cm4,60020,1351671478752",
      "2012-10-31 19:52:15,923 WARN org.apache.hadoop.hbase.util.Sleeper: We slept 229128ms instead of 100000ms, this is likely due to a long garbage collecting pause and it's usually bad, see http://hbase.apache.org/book.html#trouble.rs.runtime.zkexpired"
    ],
    "stack_traces": []
  },
  {
    "file": "HBase-7799.docx",
    "title": "HBase-7799",
    "version": "hbase-0.94.27",
    "Description": "reassigning region stuck in open still may not work correctly due to leftover ZK node",
    "logs": [
      "2013-02-08 14:35:06,500 INFO  [MASTER_SERVER_OPERATIONS-10.11.2.92,64483,1360362800340-1] master.RegionStates(347): Found opening region {IntegrationTestRebalanceAndKillServersTargeted,7333332c,1360362805563.871d1c3bdf98a2c93b527cb6cc61327d. state=OPENING, ts=1360362901596, server=10.11.2.92,64485,1360362800564} to be reassigned by SSH for 10.11.2.92,64485,1360362800564",
      "2013-02-08 14:35:06,500 INFO  [MASTER_SERVER_OPERATIONS-10.11.2.92,64483,1360362800340-1] master.RegionStates(242): Region {NAME => 'IntegrationTestRebalanceAndKillServersTargeted,7333332c,1360362805563.871d1c3bdf98a2c93b527cb6cc61327d.', STARTKEY => '7333332c', ENDKEY => '7ffffff8', ENCODED => 871d1c3bdf98a2c93b527cb6cc61327d,} transitioned from {IntegrationTestRebalanceAndKillServersTargeted,7333332c,1360362805563.871d1c3bdf98a2c93b527cb6cc61327d. state=OPENING, ts=1360362901596, server=10.11.2.92,64485,1360362800564} to {IntegrationTestRebalanceAndKillServersTargeted,7333332c,1360362805563.871d1c3bdf98a2c93b527cb6cc61327d. state=CLOSED, ts=1360362906500, server=null}",
      "2013-02-08 14:35:06,505 DEBUG [10.11.2.92,64483,1360362800340-GeneralBulkAssigner-1] master.AssignmentManager(1530): Forcing OFFLINE; was={IntegrationTestRebalanceAndKillServersTargeted,7333332c,1360362805563.871d1c3bdf98a2c93b527cb6cc61327d. state=CLOSED, ts=1360362906500, server=null}",
      "2013-02-08 14:35:06,506 DEBUG [10.11.2.92,64483,1360362800340-GeneralBulkAssigner-1] zookeeper.ZKAssign(176): master:64483-0x13cbbf1025d0000 Async create of unassigned node for 871d1c3bdf98a2c93b527cb6cc61327d with OFFLINE state",
      "2013-02-08 14:35:06,509 WARN  [main-EventThread] master.OfflineCallback(59): Node for /hbase/region-in-transition/871d1c3bdf98a2c93b527cb6cc61327d already exists",
      "2013-02-08 14:35:06,509 DEBUG [main-EventThread] master.OfflineCallback(69): rs={IntegrationTestRebalanceAndKillServersTargeted,7333332c,1360362805563.871d1c3bdf98a2c93b527cb6cc61327d. state=OFFLINE, ts=1360362906506, server=null}, server=10.11.2.92,64488,1360362800651",
      "2013-02-08 14:35:06,512 DEBUG [main-EventThread] master.OfflineCallback$ExistCallback(106): rs={IntegrationTestRebalanceAndKillServersTargeted,7333332c,1360362805563.871d1c3bdf98a2c93b527cb6cc61327d. state=OFFLINE, ts=1360362906506, server=null}, server=10.11.2.92,64488,1360362800651",
      "2013-02-08 14:35:06,517 INFO  [PRI IPC Server handler 7 on 64488] regionserver.HRegionServer(3435): Received request to open region: IntegrationTestRebalanceAndKillServersTargeted,7333332c,1360362805563.871d1c3bdf98a2c93b527cb6cc61327d. on 10.11.2.92,64488,1360362800651",
      "2013-02-08 14:35:06,521 WARN  [RS_OPEN_REGION-10.11.2.92,64488,1360362800651-0] zookeeper.ZKAssign(762): regionserver:64488-0x13cbbf1025d0004 Attempt to transition the unassigned node for 871d1c3bdf98a2c93b527cb6cc61327d from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING failed, the node existed but was in the state RS_ZK_REGION_OPENING set by the server 10.11.2.92,64488,1360362800651",
      "2013-02-08 14:35:06,528 WARN  [RS_OPEN_REGION-10.11.2.92,64488,1360362800651-0] zookeeper.ZKAssign(762): regionserver:64488-0x13cbbf1025d0004 Attempt to transition the unassigned node for 871d1c3bdf98a2c93b527cb6cc61327d from M_ZK_REGION_OFFLINE to RS_ZK_REGION_FAILED_OPEN failed, the node existed but was in the state RS_ZK_REGION_OPENING set by the server 10.11.2.92,64488,1360362800651",
      "2013-02-08 14:36:09,644 INFO  [PRI IPC Server handler 3 on 65343] regionserver.HRegionServer(3435): Received request to open region: IntegrationTestRebalanceAndKillServersTargeted,7333332c,1360362805563.871d1c3bdf98a2c93b527cb6cc61327d. on 10.11.2.92,65343,1360362907858",
      "2013-02-08 14:36:09,658 WARN  [RS_OPEN_REGION-10.11.2.92,65343,1360362907858-0] zookeeper.ZKAssign(762): regionserver:65343-0x13cbbf1025d000f Attempt to transition the unassigned node for 871d1c3bdf98a2c93b527cb6cc61327d from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING failed, the node existed but was in the state RS_ZK_REGION_OPENING set by the server 10.11.2.92,65343,1360362907858",
      "2013-02-08 14:36:09,662 WARN  [RS_OPEN_REGION-10.11.2.92,65343,1360362907858-0] zookeeper.ZKAssign(762): regionserver:65343-0x13cbbf1025d000f Attempt to transition the unassigned node for 871d1c3bdf98a2c93b527cb6cc61327d from M_ZK_REGION_OFFLINE to RS_ZK_REGION_FAILED_OPEN failed, the node existed but was in the state RS_ZK_REGION_OPENING set by the server 10.11.2.92,65343,1360362907858",
      "2013-02-08 14:38:17,696 INFO  [PRI IPC Server handler 6 on 49317] regionserver.HRegionServer(3435): Received request to open region: IntegrationTestRebalanceAndKillServersTargeted,7333332c,1360362805563.871d1c3bdf98a2c93b527cb6cc61327d. on 10.11.2.92,49317,1360362971256",
      "2013-02-08 14:38:17,713 WARN  [RS_OPEN_REGION-10.11.2.92,49317,1360362971256-2] zookeeper.ZKAssign(762): regionserver:49317-0x13cbbf1025d0011 Attempt to transition the unassigned node for 871d1c3bdf98a2c93b527cb6cc61327d from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING failed, the node existed but was in the state RS_ZK_REGION_OPENING set by the server 10.11.2.92,49317,1360362971256",
      "2013-02-08 14:38:17,720 WARN  [RS_OPEN_REGION-10.11.2.92,49317,1360362971256-2] zookeeper.ZKAssign(762): regionserver:49317-0x13cbbf1025d0011 Attempt to transition the unassigned node for 871d1c3bdf98a2c93b527cb6cc61327d from M_ZK_REGION_OFFLINE to RS_ZK_REGION_FAILED_OPEN failed, the node existed but was in the state RS_ZK_REGION_OPENING set by the server 10.11.2.92,49317,1360362971256",
      "2013-02-08 14:39:22,086 DEBUG [main-EventThread] master.OfflineCallback(69): rs={IntegrationTestRebalanceAndKillServersTargeted,0ccccccc,1360362805563.f6daa3cccefad73f8141d200ce58645b. state=OFFLINE, ts=1360363161900, server=null}, server=10.11.2.92,49476,1360363036574",
      "2013-02-08 14:39:22,091 DEBUG [HBaseWriterThread_8] client.MetaScanner(252): Current INFO from scan results = {NAME => 'IntegrationTestRebalanceAndKillServersTargeted,e6666658,1360362805564.763919a2e8ebd05f3735805ba2541ed6.', STARTKEY => 'e6666658', ENDKEY => 'f3333324', ENCODED => 763919a2e8ebd05f3735805ba2541ed6,}",
      "2013-02-08 14:39:22,091 DEBUG [HBaseWriterThread_8] client.MetaScanner(199): Scanning .META. starting at row=IntegrationTestRebalanceAndKillServersTargeted,e6666658,00000000000000 for max=10 rows using hconnection 0x610f7612",
      "2013-02-08 14:39:22,092 DEBUG [HBaseWriterThread_8] client.MetaScanner(252): Current INFO from scan results = {NAME => 'IntegrationTestRebalanceAndKillServersTargeted,e6666658,1360362805564.763919a2e8ebd05f3735805ba2541ed6.', STARTKEY => 'e6666658', ENDKEY => 'f3333324', ENCODED => 763919a2e8ebd05f3735805ba2541ed6,}",
      "2013-02-08 14:39:22,092 DEBUG [HBaseWriterThread_8] client.HConnectionManager$HConnectionImplementation(1387): Cached location for IntegrationTestRebalanceAndKillServersTargeted,e6666658,1360362805564.763919a2e8ebd05f3735805ba2541ed6. is 10.11.2.92:64486",
      "2013-02-08 14:39:22,093 DEBUG [HBaseWriterThread_8] client.MetaScanner(252): Current INFO from scan results = {NAME => 'IntegrationTestRebalanceAndKillServersTargeted,f3333324,1360362805564.d362fd7914d7e74e3cff35e8bcaa8c9f.', STARTKEY => 'f3333324', ENDKEY => '', ENCODED => d362fd7914d7e74e3cff35e8bcaa8c9f,}",
      "2013-02-08 14:39:22,093 TRACE [HBaseWriterThread_8] client.HConnectionManager$HConnectionImplementation$Process(2036): Will retry requests to [10.11.2.92:64486] after delay of [32041] for rows [ed777f62b485b3975ac2d4ca644d619e-24475;]",
      "2013-02-08 14:39:22,107 WARN  [main-EventThread] master.OfflineCallback(59): Node for /hbase/region-in-transition/871d1c3bdf98a2c93b527cb6cc61327d already exists",
      "2013-02-08 14:39:22,107 DEBUG [main-EventThread] master.OfflineCallback(69): rs={IntegrationTestRebalanceAndKillServersTargeted,7333332c,1360362805563.871d1c3bdf98a2c93b527cb6cc61327d. state=OFFLINE, ts=1360363161906, server=null}, server=10.11.2.92,49780,1360363101797",
      "2013-02-08 14:39:22,107 WARN  [main-EventThread] master.OfflineCallback(59): Node for /hbase/region-in-transition/763919a2e8ebd05f3735805ba2541ed6 already exists",
      "2013-02-08 14:39:22,107 DEBUG [main-EventThread] master.OfflineCallback(69): rs={IntegrationTestRebalanceAndKillServersTargeted,e6666658,1360362805564.763919a2e8ebd05f3735805ba2541ed6. state=OFFLINE, ts=1360363161906, server=null}, server=10.11.2.92,64486,1360362800604",
      "2013-02-08 14:39:22,117 INFO  [10.11.2.92,64483,1360362800340-GeneralBulkAssigner-1] master.AssignmentManager(1331): 10.11.2.92,49780,1360363101797 unassigned znodes=5 of total=5",
      "2013-02-08 14:39:22,117 INFO  [10.11.2.92,64483,1360362800340-GeneralBulkAssigner-1] master.RegionStates(242): Region {NAME => 'IntegrationTestRebalanceAndKillServersTargeted,7333332c,1360362805563.871d1c3bdf98a2c93b527cb6cc61327d.', STARTKEY => '7333332c', ENDKEY => '7ffffff8', ENCODED => 871d1c3bdf98a2c93b527cb6cc61327d,} transitioned from {IntegrationTestRebalanceAndKillServersTargeted,7333332c,1360362805563.871d1c3bdf98a2c93b527cb6cc61327d. state=OFFLINE, ts=1360363161906, server=null} to {IntegrationTestRebalanceAndKillServersTargeted,7333332c,1360362805563.871d1c3bdf98a2c93b527cb6cc61327d. state=PENDING_OPEN, ts=1360363162117, server=10.11.2.92,49780,1360363101797}",
      "2013-02-08 14:39:22,117 DEBUG [main-EventThread] master.OfflineCallback$ExistCallback(106): rs={IntegrationTestRebalanceAndKillServersTargeted,f3333324,1360362805564.d362fd7914d7e74e3cff35e8bcaa8c9f. state=OFFLINE, ts=1360363161913, server=null}, server=10.11.2.92,64486,1360362800604",
      "2013-02-08 14:39:22,117 INFO  [10.11.2.92,64483,1360362800340-GeneralBulkAssigner-1] master.RegionStates(242): Region {NAME => 'IntegrationTestRebalanceAndKillServersTargeted,99999990,1360362805563.569ec0c144f9033bb42e5e4936fdda4f.', STARTKEY => '99999990', ENDKEY => 'a666665c', ENCODED => 569ec0c144f9033bb42e5e4936fdda4f,} transitioned from {IntegrationTestRebalanceAndKillServersTargeted,99999990,1360362805563.569ec0c144f9033bb42e5e4936fdda4f. state=OFFLINE, ts=1360363161911, server=null} to {IntegrationTestRebalanceAndKillServersTargeted,99999990,1360362805563.569ec0c144f9033bb42e5e4936fdda4f. state=PENDING_OPEN, ts=1360363162117, server=10.11.2.92,49780,1360363101797}",
      "2013-02-08 14:39:22,118 INFO  [PRI IPC Server handler 6 on 49780] regionserver.HRegionServer(3435): Received request to open region: IntegrationTestRebalanceAndKillServersTargeted,7333332c,1360362805563.871d1c3bdf98a2c93b527cb6cc61327d. on 10.11.2.92,49780,1360363101797",
      "2013-02-08 14:39:22,121 WARN  [RS_OPEN_REGION-10.11.2.92,49780,1360363101797-1] zookeeper.ZKAssign(762): regionserver:49780-0x13cbbf1025d0015 Attempt to transition the unassigned node for 871d1c3bdf98a2c93b527cb6cc61327d from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING failed, the node existed but was in the state RS_ZK_REGION_OPENING set by the server 10.11.2.92,49780,1360363101797",
      "2013-02-08 14:39:22,135 WARN  [RS_OPEN_REGION-10.11.2.92,49780,1360363101797-1] zookeeper.ZKAssign(762): regionserver:49780-0x13cbbf1025d0015 Attempt to transition the unassigned node for 871d1c3bdf98a2c93b527cb6cc61327d from M_ZK_REGION_OFFLINE to RS_ZK_REGION_FAILED_OPEN failed, the node existed but was in the state RS_ZK_REGION_OPENING set by the server 10.11.2.92,49780,1360363101797"
    ],
    "stack_traces": [
      "java.util.concurrent.ExecutionException: org.apache.hadoop.hbase.NotServingRegionException: Region is not online: 763919a2e8ebd05f3735805ba2541ed6\n\tat java.util.concurrent.FutureTask$Sync.innerGet(FutureTask.java:222)\n\tat java.util.concurrent.FutureTask.get(FutureTask.java:83)\n\tat org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$Process.processBatchCallback(HConnectionManager.java:2113)\n\tat org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$Process.access$900(HConnectionManager.java:1950)\n\tat org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.processBatchCallback(HConnectionManager.java:1939)\n\tat org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.processBatch(HConnectionManager.java:1918)\n\tat org.apache.hadoop.hbase.client.HTable.flushCommits(HTable.java:1026)\n\tat org.apache.hadoop.hbase.client.HTable.doPut(HTable.java:695)\n\tat org.apache.hadoop.hbase.client.HTable.put(HTable.java:670)\n\tat org.apache.hadoop.hbase.util.MultiThreadedWriter$HBaseWriterThread.insert(MultiThreadedWriter.java:175)\n\tat org.apache.hadoop.hbase.util.MultiThreadedWriter$HBaseWriterThread.run(MultiThreadedWriter.java:145)\nCaused by: org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: Region is not online: 763919a2e8ebd05f3735805ba2541ed6\n\tat sun.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source)\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27)\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:513)\n\tat org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:95)\n\tat org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:79)\n\tat org.apache.hadoop.hbase.ipc.ProtobufRpcClientEngine$Invoker.invoke(ProtobufRpcClientEngine.java:148)\n\tat $Proxy20.multi(Unknown Source)\n\tat org.apache.hadoop.hbase.protobuf.ProtobufUtil.multi(ProtobufUtil.java:1092)\n\tat org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$3$1.call(HConnectionManager.java:1783)\n\tat org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$3$1.call(HConnectionManager.java:1781)\n\tat org.apache.hadoop.hbase.client.ServerCallable.withoutRetries(ServerCallable.java:219)\n\tat org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$3.call(HConnectionManager.java:1792)\n\tat org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$3.call(HConnectionManager.java:1778)\n\tat org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$Process$1.call(HConnectionManager.java:2290)\n\tat org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$Process$1.call(HConnectionManager.java:2280)\n\tat java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:138)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\n\tat java.lang.Thread.run(Thread.java:680)\nCaused by: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hbase.NotServingRegionException: Region is not online: 763919a2e8ebd05f3735805ba2541ed6\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2538)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3745)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3211)\n\tat sun.reflect.GeneratedMethodAccessor37.invoke(Unknown Source)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\n\tat java.lang.reflect.Method.invoke(Method.java:597)\n\tat org.apache.hadoop.hbase.ipc.ProtobufRpcServerEngine$Server.call(ProtobufRpcServerEngine.java:197)\n\tat org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1786)\n\tat org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:1304)\n\tat org.apache.hadoop.hbase.ipc.ProtobufRpcClientEngine$Invoker.invoke(ProtobufRpcClientEngine.java:139)"
    ]
  },
  {
    "file": "HBase-8230.docx",
    "title": "HBase-8230",
    "version": "hbase-0.94.6",
    "Description": "Possible NPE on regionserver abort if replication service has not been started",
    "logs": [
      "2013-03-29 10:32:42,251 INFO  [regionserver26003] STOPPED: Failed initialization org.apache.hadoop.hbase.regionserver.HRegionServer.stop(HRegionServer.java:1665)",
      "2013-03-29 10:32:42,253 ERROR [regionserver26003] Failed init org.apache.hadoop.hbase.regionserver.HRegionServer.cleanup(HRegionServer.java:1161)",
      "2013-03-29 10:32:42,260 FATAL [regionserver26003] ABORTING region server om-host2,26003,1364524173470: Unhandled exception: cannot get log writer org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:1737)"
    ],
    "stack_traces": [
      "Exception in thread \"regionserver26003\" java.lang.NullPointerException\n\tat org.apache.hadoop.hbase.replication.regionserver.Replication.join(Replication.java:129)\n\tat org.apache.hadoop.hbase.replication.regionserver.Replication.stopReplicationService(Replication.java:120)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.join(HRegionServer.java:1803)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:834)\n\tat java.lang.Thread.run(Thread.java:662)",
      "java.io.IOException: cannot get log writer\n\tat org.apache.hadoop.hbase.regionserver.wal.HLog.createWriter(HLog.java:757)\n\tat org.apache.hadoop.hbase.regionserver.wal.HLog.createWriterInstance(HLog.java:701)\n\tat org.apache.hadoop.hbase.regionserver.wal.HLog.rollWriter(HLog.java:637)\n\tat org.apache.hadoop.hbase.regionserver.wal.HLog.rollWriter(HLog.java:582)\n\tat org.apache.hadoop.hbase.regionserver.wal.HLog.<init>(HLog.java:436)\n\tat org.apache.hadoop.hbase.regionserver.wal.HLog.<init>(HLog.java:362)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.instantiateHLog(HRegionServer.java:1327)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.setupWALAndReplication(HRegionServer.java:1316)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.handleReportForDutyResponse(HRegionServer.java:1030)\n\tat org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:706)\n\tat java.lang.Thread.run(Thread.java:662)\nCaused by: java.io.IOException: org.apache.hadoop.hdfs.server.namenode.SafeModeException: Cannot create file/hbase/.logs/om-host2,26003,1364524173470/om-host2%2C26003%2C1364524173470.1364524361366. Name node is in safe mode.\nThe reported blocks 14 has reached the threshold 0.9990 of total blocks 14. Safe mode will be turned off automatically in 21 seconds.\n\tat org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInternal(FSNamesystem.java:1601)\n\tat org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFile(FSNamesystem.java:1547)\n\tat org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.create(NameNodeRpcServer.java:412)\n\tat org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.create(ClientNamenodeProtocolServerSideTranslatorPB.java:204)\n\tat org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java:43664)\n\tat org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:427)\n\tat org.apache.hadoop.ipc.RPC$Server.call(RPC.java:924)\n\tat org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1710)\n\tat org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1706)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat javax.security.auth.Subject.doAs(Subject.java:396)\n\tat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1232)\n\tat org.apache.hadoop.ipc.Server$Handler.run(Server.java:1704)\n\tat org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.init(SequenceFileLogWriter.java:209)\n\tat org.apache.hadoop.hbase.regionserver.wal.HLog.createWriter(HLog.java:754)"
    ]
  },
  {
    "file": "HBase-8519.docx",
    "title": "HBase-8519",
    "version": "hbase-0.94.7",
    "Description": "Backup master will never come up if primary master dies during initialization",
    "logs": [
      "2013-05-09 15:08:05,568 INFO org.apache.hadoop.hbase.master.metrics.MasterMetrics: Initialized",
      "2013-05-09 15:08:05,573 DEBUG org.apache.hadoop.hbase.master.HMaster: HMaster started in backup mode. Stalling until master znode is written.",
      "2013-05-09 15:08:05,589 INFO org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper: Node /hbase/master already exists and this is not a retry",
      "2013-05-09 15:08:05,590 INFO org.apache.hadoop.hbase.master.ActiveMasterManager: Adding ZNode for /hbase/backup-masters/xxx.com,60000,1368137285373 in backup master directory",
      "2013-05-09 15:08:05,595 INFO org.apache.hadoop.hbase.master.ActiveMasterManager: Another master is the active master, xxx.com,60000,1368137283107; waiting to become the next active master",
      "2013-05-09 15:09:45,006 DEBUG org.apache.hadoop.hbase.master.ActiveMasterManager: No master available. Notifying waiting threads",
      "2013-05-09 15:09:45,006 INFO org.apache.hadoop.hbase.master.HMaster: Cluster went down before this master became active",
      "2013-05-09 15:09:45,006 DEBUG org.apache.hadoop.hbase.master.HMaster: Stopping service threads",
      "2013-05-09 15:09:45,006 INFO org.apache.hadoop.ipc.HBaseServer: Stopping server on 60000",
      "2013-05-10 18:20:17,690 INFO org.apache.hadoop.hbase.master.metrics.MasterMetrics: Initialized",
      "2013-05-10 18:20:17,704 INFO org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper: Node /hbase/master already exists and this is not a retry",
      "2013-05-10 18:20:17,705 INFO org.apache.hadoop.hbase.master.ActiveMasterManager: Adding ZNode for /hbase/backup-masters/xxx.com,60000,1368235217501 in backup master directory",
      "2013-05-10 18:20:17,711 INFO org.apache.hadoop.hbase.master.ActiveMasterManager: Current master has this master's address, xxx.com,60000,1368234912934; master was restarted? Deleting node.",
      "2013-05-10 18:20:17,711 DEBUG org.apache.hadoop.hbase.master.ActiveMasterManager: No master available. Notifying waiting threads",
      "2013-05-10 18:20:17,711 INFO org.apache.hadoop.hbase.master.HMaster: Cluster went down before this master became active",
      "2013-05-10 18:20:17,711 DEBUG org.apache.hadoop.hbase.master.HMaster: Stopping service threads"
    ],
    "stack_traces": []
  },
  {
    "file": "HBase-9387.docx",
    "title": "HBase-9387",
    "version": "hbase-0.95.2",
    "Description": "Region could get lost during assignment",
    "logs": [
      "2013-08-29 22:15:34,180 INFO  [AM.ZK.Worker-pool2-t4] master.RegionStates(299): Onlined 1588230740 on kiyo.gq1.ygridcore.net,57016,1377814510039",
      "2013-08-29 22:15:34,587 DEBUG [Thread-221] client.HConnectionManager$HConnectionImplementation(1269): locateRegionInMeta parentTable=hbase:meta, metaLocation={region=hbase:meta,,1.1588230740, hostname=kiyo.gq1.ygridcore.net,57016,1377814510039, seqNum=0}, attempt=2 of 35 failed; retrying after sleep of 302 because: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region is being opened: 1588230740",
      "2013-08-29 22:15:29,978 DEBUG [RS_OPEN_META-kiyo:57016-0] zookeeper.ZKAssign(786): regionserver:57016-0x140cc24e86a0003 Transitioning 1588230740 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED",
      "2013-08-29 22:15:32,746 WARN  [RS_OPEN_META-kiyo:57016-0] zookeeper.RecoverableZooKeeper(258): Possibly transient ZooKeeper, quorum=localhost:61251, exception=org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss for /hbase/region-in-transition/1588230740",
      "2013-08-29 22:15:32,746 INFO [RS_OPEN_META-kiyo:57016-0] util.RetryCounter(54): Sleeping 2000ms before retry #1...",
      "2013-08-29 22:15:34,148 DEBUG [AM.ZK.Worker-pool2-t4] master.AssignmentManager(801): Handling transition=RS_ZK_REGION_OPENED, server=kiyo.gq1.ygridcore.net,57016,1377814510039, region=1588230740, current state from region state map ={1588230740 state=OPENING, ts=1377814529578, server=kiyo.gq1.ygridcore.net,57016,1377814510039}",
      "2013-08-29 22:15:34,149 DEBUG [AM.ZK.Worker-pool2-t4] zookeeper.ZKAssign(405): master:44745-0x140cc24e86a0001-0x140cc24e86a0001 Deleting existing unassigned node 1588230740 in expected state RS_ZK_REGION_OPENED",
      "2013-08-29 22:15:34,179 DEBUG [pool-1-thread-1-EventThread] zookeeper.ZooKeeperWatcher(312): master:44745-0x140cc24e86a0001-0x140cc24e86a0001 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/region-in-transition/1588230740",
      "2013-08-29 22:15:34,760 WARN [RS_OPEN_META-kiyo:57016-0] zookeeper.ZKAssign(866): regionserver:57016-0x140cc24e86a0003-0x140cc24e86a0003 Attempt to transition the unassigned node for 1588230740 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED failed, the node existed and was in the expected state but then when setting data it no longer existed",
      "2013-08-29 22:15:34,761 WARN [RS_OPEN_META-kiyo:57016-0] handler.OpenRegionHandler(370): Completed the OPEN of region hbase:meta,,1.1588230740 but when transitioning from OPENING to OPENED got a version mismatch, someone else clashed so now unassigning -- closing region on server: kiyo.gq1.ygridcore.net,57016,1377814510039",
      "2013-08-29 22:15:34,761 DEBUG [RS_OPEN_META-kiyo:57016-0] regionserver.HRegion(950): Closing hbase:meta,,1.1588230740: disabling compactions & flushes",
      "2013-08-29 22:15:34,761 DEBUG [RS_OPEN_META-kiyo:57016-0] regionserver.HRegion(972): Updates disabled for region hbase:meta,,1.1588230740",
      "2013-08-29 22:15:34,762 INFO [StoreCloserThread-hbase:meta,,1.1588230740-1] regionserver.HStore(668): Closed info",
      "2013-08-29 22:15:34,762 INFO [RS_OPEN_META-kiyo:57016-0] regionserver.HRegion(1030): Closed hbase:meta,,1.1588230740",
      "2013-08-29 22:15:34,763 INFO [RS_OPEN_META-kiyo:57016-0] handler.OpenRegionHandler(396): Opening of region {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} failed, transitioning from OPENING to FAILED_OPEN in ZK, expecting version 1",
      "2013-08-29 22:15:34,763 DEBUG [RS_OPEN_META-kiyo:57016-0] zookeeper.ZKAssign(786): regionserver:57016-0x140cc24e86a0003-0x140cc24e86a0003 Transitioning 1588230740 from RS_ZK_REGION_OPENING to RS_ZK_REGION_FAILED_OPEN",
      "2013-08-29 22:15:34,765 DEBUG [RS_OPEN_META-kiyo:57016-0] zookeeper.ZKUtil(784): regionserver:57016-0x140cc24e86a0003-0x140cc24e86a0003 Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error)",
      "2013-08-29 22:15:34,765 WARN [RS_OPEN_META-kiyo:57016-0] handler.OpenRegionHandler(404): Unable to mark region {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} as FAILED_OPEN. It's likely that the master already timed out this open attempt, and thus another RS already has the region."
    ],
    "stack_traces": [
      "org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region is being opened: 1588230740\n\t\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2574)\n\t\tat org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3949)\n\t\tat org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2733)\n\t\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:26965)\n\t\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2063)\n\t\tat org.apache.hadoop.hbase.ipc.RpcServer$CallRunner.run(RpcServer.java:1800)\n\t\tat org.apache.hadoop.hbase.ipc.SimpleRpcScheduler.consumerLoop(SimpleRpcScheduler.java:165)\n\t\tat org.apache.hadoop.hbase.ipc.SimpleRpcScheduler.access$000(SimpleRpcScheduler.java:41)"
    ]
  },
  {
    "file": "HBase-9593.docx",
    "title": "HBase-9593",
    "version": "hbase-0.94.11",
    "Description": "Region server left in online servers list forever if it went down after registering to master and before creating ephemeral node.",
    "logs": [
      "2013-09-19 19:47:41,123 DEBUG org.apache.hadoop.hbase.master.ServerManager: STARTUP: Server HOST-10-18-40-153,61020,1379600260255 came back up, removed it from the dead servers list",
      "2013-09-19 19:47:41,123 INFO org.apache.hadoop.hbase.master.ServerManager: Registering server=HOST-10-18-40-153,61020,1379600260255",
      "2013-09-19 19:47:41,119 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: Connected to master at HOST-10-18-40-153/10.18.40.153:61000",
      "2013-09-19 19:47:41,119 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: Telling master at HOST-10-18-40-153,61000,1379600055284 that we are up with port=61020, startcode=1379600260255",
      "2013-09-19 19:47:54,049 WARN org.apache.hadoop.hbase.master.AssignmentManager: Failed assignment of -ROOT-,,0.70236052 to HOST-10-18-40-153,61020,1379600260255, trying to assign elsewhere instead; retry=0",
      "2013-09-19 19:47:54,050 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Found an existing plan for -ROOT-,,0.70236052 destination server is HOST-10-18-40-153,61020,1379600260255",
      "2013-09-19 19:47:54,050 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: No previous transition plan was found (or we are ignoring an existing plan) for -ROOT-,,0.70236052 so generated a random one; hri=-ROOT-,,0.70236052, src=, dest=HOST-10-18-40-153,61020,1379600260255; 1 (online=1, available=1) available servers",
      "2013-09-19 19:47:54,050 DEBUG org.apache.hadoop.hbase.zookeeper.ZKAssign: master:61000-0x14135a277ff017d Creating (or updating) unassigned node for 70236052 with OFFLINE state",
      "2013-09-19 19:47:54,070 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Handling transition=M_ZK_REGION_OFFLINE, server=HOST-10-18-40-153,61000,1379600055284, region=70236052/-ROOT-",
      "2013-09-19 19:47:54,071 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Found an existing plan for -ROOT-,,0.70236052 destination server is HOST-10-18-40-153,61020,1379600260255",
      "2013-09-19 19:47:54,071 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Using pre-existing plan for region -ROOT-,,0.70236052; plan=hri=-ROOT-,,0.70236052, src=, dest=HOST-10-18-40-153,61020,1379600260255",
      "2013-09-19 19:47:54,071 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Assigning region -ROOT-,,0.70236052 to HOST-10-18-40-153,61020,1379600260255",
      "2013-09-19 19:47:54,072 WARN org.apache.hadoop.hbase.master.AssignmentManager: Failed assignment of -ROOT-,,0.70236052 to HOST-10-18-40-153,61020,1379600260255, trying to assign elsewhere instead; retry=1",
      "2013-09-19 19:47:54,072 DEBUG org.apache.hadoop.hbase.master.AssignmentManager: Found an existing plan for -ROOT-,,0.70236052 destination server is HOST-10-18-40-153,61020,1379600260255"
    ],
    "stack_traces": [
      "java.net.ConnectException: Connection refused\n\tat sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)\n\tat sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:567)\n\tat org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)\n\tat org.apache.hadoop.net.NetUtils.connect(NetUtils.java:529)\n\tat org.apache.hadoop.net.NetUtils.connect(NetUtils.java:493)\n\tat org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupConnection(HBaseClient.java:390)\n\tat org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:436)\n\tat org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:1127)\n\tat org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:974)\n\tat org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)\n\tat $Proxy15.openRegion(Unknown Source)\n\tat org.apache.hadoop.hbase.master.ServerManager.sendRegionOpen(ServerManager.java:533)\n\tat org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1734)\n\tat org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1431)\n\tat org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1406)\n\tat org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1401)\n\tat org.apache.hadoop.hbase.master.AssignmentManager.assignRoot(AssignmentManager.java:2374)\n\tat org.apache.hadoop.hbase.master.handler.MetaServerShutdownHandler.verifyAndAssignRoot(MetaServerShutdownHandler.java:136)\n\tat org.apache.hadoop.hbase.master.handler.MetaServerShutdownHandler.verifyAndAssignRootWithRetries(MetaServerShutdownHandler.java:160)\n\tat org.apache.hadoop.hbase.master.handler.MetaServerShutdownHandler.process(MetaServerShutdownHandler.java:82)\n\tat org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:175)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\n\tat java.lang.Thread.run(Thread.java:662)",
      "org.apache.hadoop.hbase.ipc.HBaseClient$FailedServerException: This server is in the failed servers list: HOST-10-18-40-153/10.18.40.153:61020\n\tat org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:425)\n\tat org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:1127)\n\tat org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:974)\n\tat org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)\n\tat $Proxy15.openRegion(Unknown Source)\n\tat org.apache.hadoop.hbase.master.ServerManager.sendRegionOpen(ServerManager.java:533)\n\tat org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1734)\n\tat org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1431)\n\tat org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1406)\n\tat org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1401)\n\tat org.apache.hadoop.hbase.master.AssignmentManager.assignRoot(AssignmentManager.java:2374)\n\tat org.apache.hadoop.hbase.master.handler.MetaServerShutdownHandler.verifyAndAssignRoot(MetaServerShutdownHandler.java:136)\n\tat org.apache.hadoop.hbase.master.handler.MetaServerShutdownHandler.verifyAndAssignRootWithRetries(MetaServerShutdownHandler.java:160)\n\tat org.apache.hadoop.hbase.master.handler.MetaServerShutdownHandler.process(MetaServerShutdownHandler.java:82)\n\tat org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:175)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\n\tat java.lang.Thread.run(Thread.java:662)"
    ]
  },
  {
    "file": "HBase-9740.docx",
    "title": "HBase-9740",
    "version": "hbase-0.94.16",
    "Description": "A corrupt HFile could cause endless attempts to assign the region without a chance of success",
    "logs": [],
    "stack_traces": [
      "java.io.IOException: java.io.IOException: org.apache.hadoop.hbase.io.hfile.CorruptHFileException: Problem reading HFile Trailer from file /hbase/userdata/318c533716869574f10615703269497f/data/a3e2ae39f71441ac92a6563479fb976e\n\tat org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionInternals(HRegion.java:550)\n\tat org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:463)\n\tat org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:3835)\n\tat org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:3783)\n\tat org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.openRegion(OpenRegionHandler.java:332)\n\tat org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.process(OpenRegionHandler.java:108)\n\tat org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:169)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\n\tat java.lang.Thread.run(Thread.java:662)\nCaused by: java.io.IOException: org.apache.hadoop.hbase.io.hfile.CorruptHFileException: Problem reading HFile Trailer from file /hbase/userdata/318c533716869574f10615703269497f/data/a3e2ae39f71441ac92a6563479fb976e\n\tat org.apache.hadoop.hbase.regionserver.Store.loadStoreFiles(Store.java:404)\n\tat org.apache.hadoop.hbase.regionserver.Store.<init>(Store.java:257)\n\tat org.apache.hadoop.hbase.regionserver.HRegion.instantiateHStore(HRegion.java:3017)\n\tat org.apache.hadoop.hbase.regionserver.HRegion$1.call(HRegion.java:525)\n\tat org.apache.hadoop.hbase.regionserver.HRegion$1.call(HRegion.java:523)\n\tat java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:138)\n\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)\n\tat java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:138)"
    ]
  },
  {
    "file": "HBase-9773.docx",
    "title": "HBase-9773",
    "version": "hbase-0.96.0",
    "Description": "Master aborted when hbck asked the master to assign a region that was already online",
    "logs": [
      "2013-10-12 10:42:57,067|beaver.machine|INFO|ERROR: Region { meta => hbase:namespace,,1381564449706.a0ac0825ba2d0830614e7f808f31787a., hdfs => hdfs://gs-hdp2-secure-1381559462-hbase-12.cs1cloud.internal:8020/apps/hbase/data/data/hbase/namespace/a0ac0825ba2d0830614e7f808f31787a, deployed => } not deployed on any region server.",
      "2013-10-12 10:42:57,067|beaver.machine|INFO|Trying to fix unassigned region...",
      "2013-10-12 10:52:35,960 INFO [RpcServer.handler=4,port=60000] master.HMaster: Client=hbase//172.18.145.105 assign hbase:namespace,,1381564449706.a0ac0825ba2d0830614e7f808f31787a.",
      "2013-10-12 10:52:35,981 DEBUG [RpcServer.handler=4,port=60000] master.AssignmentManager: Sent CLOSE to gs-hdp2-secure-1381559462-hbase-1.cs1cloud.internal,60020,1381564439794 for region hbase:namespace,,1381564449706.a0ac0825ba2d0830614e7f808f31787a.",
      "2013-10-12 10:52:36,025 DEBUG [RpcServer.handler=4,port=60000] master.AssignmentManager: No previous transition plan found (or ignoring an existing plan) for hbase:namespace,,1381564449706.a0ac0825ba2d0830614e7f808f31787a.; generated random plan=hri=hbase:namespace,,1381564449706.a0ac0825ba2d0830614e7f808f31787a., src=, dest=gs-hdp2-secure-1381559462-hbase-9.cs1cloud.internal,60020,1381564439807; 4 (online=4, available=4) available servers, forceNewPlan=true",
      "2013-10-12 10:52:36,026 FATAL [RpcServer.handler=4,port=60000] master.HMaster: Master server abort: loaded coprocessors are: [org.apache.hadoop.hbase.security.access.AccessController]",
      "2013-10-12 10:52:36,027 FATAL [RpcServer.handler=4,port=60000] master.HMaster: Unexpected state : {a0ac0825ba2d0830614e7f808f31787a state=OPEN, ts=1381564451344, server=gs-hdp2-secure-1381559462-hbase-1.cs1cloud.internal,60020,1381564439794} .. Cannot transit it to OFFLINE."
    ],
    "stack_traces": []
  },
  {
    "file": "HBase-9821.docx",
    "title": "HBase-9821",
    "version": "hbase-0.96.0",
    "Description": "Scanner id could collide",
    "logs": [
      "2013-10-21 22:43:09,037 INFO [RpcServer.handler=27,port=36020] regionserver.HRegionServer: Client tried to access missing scanner 6621035169671703961",
      "2013-10-21 22:43:09,011 ERROR [RpcServer.handler=20,port=36020] regionserver.HRegionServer: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 18 But the nextCallSeq got from client: 4470; request=scanner_id: 848804760654927372 number_of_rows: 100 close_scanner: false next_call_seq: 4470",
      "2013-10-21 22:43:09,000 INFO [RpcServer.handler=25,port=36020] regionserver.HRegionServer: Client tried to access missing scanner 4162107982028594792"
    ],
    "stack_traces": [
      "org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 18 But the nextCallSeq got from client: 4470; request=scanner_id: 848804760654927372 number_of_rows: 100 close_scanner: false next_call_seq: 4470\n\t\tat org.apache.hadoop.hbase.regionserver.HRegionServer.scan(HRegionServer.java:3030)\n\t\tat org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:27022)\n\t\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:1979)\n\t\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:90)\n\t\tat org.apache.hadoop.hbase.ipc.SimpleRpcScheduler.consumerLoop(SimpleRpcScheduler.java:160)\n\t\tat org.apache.hadoop.hbase.ipc.SimpleRpcScheduler.access$000(SimpleRpcScheduler.java:38)\n\t\tat org.apache.hadoop.hbase.ipc.SimpleRpcScheduler$1.run(SimpleRpcScheduler.java:110)\n\t\tat java.lang.Thread.run(Thread.java:724)"
    ]
  },
  {
    "file": "HDFS-1024.docx",
    "title": "HDFS-1024",
    "version": "hadoop-0.20.1",
    "Description": "SecondaryNamenode fails to checkpoint because namenode fails with CancelledKeyException",
    "logs": [
      "WARN org.mortbay.log: Committed before 410 GetImage failed. java.nio.channels.CancelledKeyException"
    ],
    "stack_traces": [
      "java.nio.channels.CancelledKeyException\nat sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:55)\nat sun.nio.ch.SelectionKeyImpl.interestOps(SelectionKeyImpl.java:59)\nat org.mortbay.io.nio.SelectChannelEndPoint.updateKey(SelectChannelEndPoint.java:324)\nat org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:278)\nat org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:542)\nat org.mortbay.jetty.AbstractGenerator$Output.flush(AbstractGenerator.java:569)\nat org.mortbay.jetty.HttpConnection$Output.flush(HttpConnection.java:946)\nat org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:646)\nat org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:577)\nat org.apache.hadoop.hdfs.server.namenode.TransferFsImage.getFileServer(TransferFsImage.java:127)\nat org.apache.hadoop.hdfs.server.namenode.GetImageServlet.doGet(GetImageServlet.java:49)\nat javax.servlet.http.HttpServlet.service(HttpServlet.java:707)\nat javax.servlet.http.HttpServlet.service(HttpServlet.java:820)\nat org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:502)\nat org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:363)\nat org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)\nat org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)\nat org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)\nat org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)\nat org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230)\nat org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)\nat org.mortbay.jetty.Server.handle(Server.java:324)\nat org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)\nat org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:864)\nat org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)\nat org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)\nat org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)\nat org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:409)\nat org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:522)"
    ]
  },
  {
    "file": "HDFS-1371.docx",
    "title": "HDFS-1371",
    "version": "hadoop-0.20.1",
    "Description": "One bad node can incorrectly flag many files as corrupt",
    "logs": [
      "2010-08-31 10:47:56,258 INFO org.apache.hadoop.hdfs.StateChange: BLOCK NameSystem.addToCorruptReplicasMap: blk_-1426587446408804113 added as corrupt on ZZ.YY.XX..220:1004 by /ZZ.YY.XX.246",
      "2010-08-31 10:47:56,290 INFO org.apache.hadoop.hdfs.StateChange: BLOCK NameSystem.addToCorruptReplicasMap: blk_-1426587446408804113 added as corrupt on ZZ.YY.XX..252:1004 by /ZZ.YY.XX.246",
      "2010-08-31 10:47:56,489 INFO org.apache.hadoop.hdfs.StateChange: BLOCK NameSystem.addToCorruptReplicasMap: blk_-1426587446408804113 added as corrupt on ZZ.YY.XX..107:1004 by /ZZ.YY.XX.246",
      "2010-08-31 10:49:00,508 INFO org.apache.hadoop.hdfs.StateChange: BLOCK NameSystem.addToCorruptReplicasMap: duplicate requested for blk_-1426587446408804113 to add as corrupt on ZZ.YY.XX.252:1004 by /ZZ.YY.XX.246",
      "2010-08-31 10:49:00,554 INFO org.apache.hadoop.hdfs.StateChange: BLOCK NameSystem.addToCorruptReplicasMap: duplicate requested for blk_-1426587446408804113 to add as corrupt on ZZ.YY.XX.107:1004 by /ZZ.YY.XX.246",
      "2010-08-31 10:49:03,934 INFO org.apache.hadoop.hdfs.StateChange: BLOCK NameSystem.addToCorruptReplicasMap: duplicate requested for blk_-1426587446408804113 to add as corrupt on ZZ.YY.XX.220:1004 by /ZZ.YY.XX.246",
      "org.apache.hadoop.fs.ChecksumException: Checksum error: /blk_-1426587446408804113:of:/myfile/part-00145.gz at 222720",
      "2010-08-31 10:47:56,256 WARN org.apache.hadoop.hdfs.DFSClient: Found Checksum error for blk_-1426587446408804113_970819282 from ZZ.YY.XX.220:1004 at 222720",
      "org.apache.hadoop.fs.ChecksumException: Checksum error: /blk_-1426587446408804113:of:/myfile/part-00145.gz at 103936",
      "2010-08-31 10:47:56,284 WARN org.apache.hadoop.hdfs.DFSClient: Found Checksum error for blk_-1426587446408804113_970819282 from ZZ.YY.XX.252:1004 at 103936",
      "org.apache.hadoop.fs.ChecksumException: Checksum error: /blk_-1426587446408804113:of:/myfile/part-00145.gz at 250368",
      "2010-08-31 10:47:56,464 WARN org.apache.hadoop.hdfs.DFSClient: Found Checksum error for blk_-1426587446408804113_970819282 from ZZ.YY.XX.107:1004 at 250368",
      "2010-08-31 10:47:56,490 INFO org.apache.hadoop.hdfs.DFSClient: Could not obtain block blk_-1426587446408804113_970819282 from any node: java.io.IOException: No live nodes contain current block. Will get new block locations from namenode and retry..."
    ],
    "stack_traces": []
  },
  {
    "file": "HDFS-2905.docx",
    "title": "HDFS-2905",
    "version": "hadoop-0.23.3",
    "Description": "HA: Standby NN NPE when shared edits dir is deleted",
    "logs": [
      "12/02/06 14:41:27 WARN namenode.FSNamesystem: NameNode low on available disk space. Already in safe mode.",
      "12/02/06 14:48:00 INFO hdfs.StateChange: STATE* Safe mode is ON. Resources are low on NN. Safe mode must be turned off manually.",
      "12/02/06 14:48:05 WARN namenode.NameNodeResourceChecker: Space available on volume '/dev/disk0s2' is 0, which is below the configured reserved amount 104857600",
      "12/02/06 14:48:05 WARN namenode.FSNamesystem: NameNode low on available disk space. Already in safe mode.",
      "12/02/06 14:48:05 INFO hdfs.StateChange: STATE* Safe mode is ON. Resources are low on NN. Safe mode must be turned off manually.",
      "12/02/06 14:48:09 ERROR ha.EditLogTailer: Unknown error encountered while tailing edits. Shutting down standby NN.",
      "12/02/06 14:48:09 INFO namenode.NameNode: SHUTDOWN_MSG:"
    ],
    "stack_traces": [
      "java.lang.NullPointerException\nat org.apache.hadoop.hdfs.server.namenode.FileJournalManager.matchEditLogs(FileJournalManager.java:160)\nat org.apache.hadoop.hdfs.server.namenode.FileJournalManager.getLogFiles(FileJournalManager.java:321)\nat org.apache.hadoop.hdfs.server.namenode.FileJournalManager.getNumberOfTransactions(FileJournalManager.java:228)\nat org.apache.hadoop.hdfs.server.namenode.JournalSet.getInputStream(JournalSet.java:216)\nat org.apache.hadoop.hdfs.server.namenode.FSEditLog.selectInputStreams(FSEditLog.java:1074)\nat org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer.doTailEdits(EditLogTailer.java:204)\nat org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer.access$600(EditLogTailer.java:57)\nat org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$EditLogTailerThread.run(EditLogTailer.java:298)"
    ]
  },
  {
    "file": "HDFS-3157.docx",
    "title": "HDFS-3157",
    "version": "hadoop-2.0.0-alpha",
    "Description": "Error in deleting block is keep on coming from DN even after the block report and directory scanning has happened",
    "logs": [
      "2012-03-19 13:41:36,905 INFO org.apache.hadoop.hdfs.StateChange: BLOCK NameSystem.addToCorruptReplicasMap: duplicate requested for blk_2903555284838653156 to add as corrupt on XX.XX.XX.XX by /XX.XX.XX.XX because reported RBW replica with genstamp 1002 does not match COMPLETE block's genstamp in block map 1003",
      "2012-03-19 13:41:39,588 INFO org.apache.hadoop.hdfs.StateChange: BLOCK* Removing block blk_2903555284838653156_1003 from neededReplications as it has enough replicas.",
      "2012-02-29 13:54:13,126 WARN org.apache.hadoop.hdfs.server.datanode.DataNode: Unexpected error trying to delete block blk_2903555284838653156_1003. BlockInfo not found in volumeMap.",
      "2012-02-29 13:54:13,126 WARN org.apache.hadoop.hdfs.server.datanode.DataNode: Error processing datanode Command",
      "2012-05-09 12:30:04,122 INFO datanode.DataNode (DataXceiver.java:writeBlock(495)) - opWriteBlock BP-2087796974-10.10.11.90-1336591801017:blk_-571802999240948417_1003 received exception org.apache.hadoop.hdfs.server.datanode.ReplicaAlreadyExistsException: Block BP-2087796974-10.10.11.90-1336591801017:blk_-571802999240948417_1003 already exists in state RBW and thus cannot be created.",
      "2012-05-09 16:05:29,960 INFO hdfs.StateChange (BlockManager.java:markBlockAsCorrupt(926)) - BLOCK markBlockAsCorrupt: block blk_-6891652617965059210_1002 could not be marked as corrupt as it does not belong to any file",
      "2012-05-12 23:57:33,773 INFO hdfs.StateChange (BlockManager.java:computeReplicationWorkForBlocks(1226)) - BLOCK* ask 127.0.0.1:54681 to replicate blk_3471690017167574595_1003 to datanode(s) 127.0.0.1:54041",
      "2012-05-12 23:57:33,791 INFO datanode.DataNode (DataNode.java:transferBlock(1221)) - DatanodeRegistration(127.0.0.1, storageID=DS-1047816814-192.168.44.128-54681-1336847251649, infoPort=62840, ipcPort=26036, storageInfo=lv=-40;cid=testClusterID;nsid=1646783488;c=0) Starting thread to transfer block BP-1770179175-192.168.44.128-1336847247907:blk_3471690017167574595_1003 to 127.0.0.1:54041",
      "2012-05-12 23:57:33,795 INFO hdfs.StateChange (BlockManager.java:processReport(1450)) - BLOCK* processReport: from DatanodeRegistration(127.0.0.1, storageID=DS-1047816814-192.168.44.128-54681-1336847251649, infoPort=62840, ipcPort=26036, storageInfo=lv=-40;cid=testClusterID;nsid=1646783488;c=0), blocks: 1, processing time: 0 msecs",
      "2012-05-12 23:57:33,796 INFO datanode.DataNode (BPServiceActor.java:blockReport(404)) - BlockReport of 1 blocks took 0 msec to generate and 2 msecs for RPC and NN processing",
      "2012-05-12 23:57:33,796 INFO datanode.DataNode (BPServiceActor.java:blockReport(423)) - sent block report, processed command:org.apache.hadoop.hdfs.server.protocol.FinalizeCommand@12eb0b3",
      "2012-05-12 23:57:33,811 INFO datanode.DataNode (DataXceiver.java:writeBlock(342)) - Receiving block BP-1770179175-192.168.44.128-1336847247907:blk_3471690017167574595_1003 src: /127.0.0.1:33583 dest: /127.0.0.1:54041",
      "2012-05-12 23:57:33,812 INFO datanode.DataNode (DataXceiver.java:writeBlock(495)) - opWriteBlock BP-1770179175-192.168.44.128-1336847247907:blk_3471690017167574595_1003 received exception org.apache.hadoop.hdfs.server.datanode.ReplicaAlreadyExistsException: Block BP-1770179175-192.168.44.128-1336847247907:blk_3471690017167574595_1003 already exists in state RBW and thus cannot be created.",
      "2012-05-12 23:57:33,814 ERROR datanode.DataNode (DataXceiver.java:run(193)) - 127.0.0.1:54041:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33583 dest: /127.0.0.1:54041",
      "2012-05-12 23:57:33,815 INFO datanode.DataNode (DataNode.java:run(1406)) - DataTransfer: Transmitted BP-1770179175-192.168.44.128-1336847247907:blk_3471690017167574595_1003 (numBytes=100) to /127.0.0.1:54041",
      "2012-05-12 23:57:34,066 INFO hdfs.StateChange (BlockManager.java:processReport(1450)) - BLOCK* processReport: from DatanodeRegistration(127.0.0.1, storageID=DS-610636930-192.168.44.128-20029-1336847250644, infoPort=52843, ipcPort=46734, storageInfo=lv=-40;cid=testClusterID;nsid=1646783488;c=0), blocks: 0, processing time: 0 msecs",
      "2012-05-12 23:57:34,067 INFO datanode.DataNode (BPServiceActor.java:blockReport(404)) - BlockReport of 0 blocks took 0 msec to generate and 3 msecs for RPC and NN processing",
      "2012-05-12 23:57:34,068 INFO datanode.DataNode (BPServiceActor.java:blockReport(423)) - sent block report, processed command:org.apache.hadoop.hdfs.server.protocol.FinalizeCommand@a1364a",
      "2012-05-12 23:57:34,099 INFO hdfs.StateChange (CorruptReplicasMap.java:addToCorruptReplicasMap(66)) - BLOCK NameSystem.addToCorruptReplicasMap: blk_3471690017167574595 added as corrupt on 127.0.0.1:54041 by /127.0.0.1 because reported RBW replica with genstamp 1002 does not match COMPLETE block's genstamp in block map 1003",
      "2012-05-12 23:57:34,100 INFO hdfs.StateChange (BlockManager.java:processReport(1450)) - BLOCK* processReport: from DatanodeRegistration(127.0.0.1, storageID=DS-1452741455-192.168.44.128-54041-1336847250645, infoPort=10314, ipcPort=16230, storageInfo=lv=-40;cid=testClusterID;nsid=1646783488;c=0), blocks: 1, processing time: 2 msecs",
      "2012-05-12 23:57:34,101 INFO datanode.DataNode (BPServiceActor.java:blockReport(404)) - BlockReport of 1 blocks took 0 msec to generate and 4 msecs for RPC and NN processing",
      "2012-05-12 23:57:34,101 INFO datanode.DataNode (BPServiceActor.java:blockReport(423)) - sent block report, processed command:org.apache.hadoop.hdfs.server.protocol.FinalizeCommand@17194a4",
      "2012-05-12 23:57:34,775 INFO hdfs.StateChange (BlockManager.java:computeReplicationWorkForBlocks(1096)) - BLOCK* Removing block blk_3471690017167574595_1003 from neededReplications as it has enough replicas."
    ],
    "stack_traces": [
      "java.io.IOException: Error in deleting blocks.\nat org.apache.hadoop.hdfs.server.datanode.FSDataset.invalidate(FSDataset.java:2061)\nat org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:581)\nat org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:545)\nat org.apache.hadoop.hdfs.server.datanode.BPServiceActor.processCommand(BPServiceActor.java:690)\nat org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:522)\nat org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:662)\nat java.lang.Thread.run(Thread.java:619)",
      "org.apache.hadoop.hdfs.server.datanode.ReplicaAlreadyExistsException: Block BP-1770179175-192.168.44.128-1336847247907:blk_3471690017167574595_1003 already exists in state RBW and thus cannot be created.\nat org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.createTemporary(FsDatasetImpl.java:795)\nat org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.createTemporary(FsDatasetImpl.java:1)\nat org.apache.hadoop.hdfs.server.datanode.BlockReceiver.<init>(BlockReceiver.java:151)\nat org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:365)\nat org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:98)\nat org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:66)\nat org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:189)\nat java.lang.Thread.run(Thread.java:619)"
    ]
  },
  {
    "file": "HDFS-3436.docx",
    "title": "HDFS-3436",
    "version": "hadoop-2.0.0-alpha",
    "Description": "adding new datanode to existing pipeline fails in case of Append/Recovery",
    "logs": [
      "2012-04-24 22:06:09,947 INFO hdfs.DFSClient (DFSOutputStream.java:createBlockOutputStream(1063)) - Exception in createBlockOutputStream",
      "2012-04-24 22:06:09,947 WARN hdfs.DFSClient (DFSOutputStream.java:setupPipelineForAppendOrRecovery(916)) - Error Recovery for block BP-1023239-10.18.40.233-1335275282109:blk_296651611851855249_1253 in pipeline *****:50010, ******:50010, *****:50010: bad datanode ******:50010  (DN3)",
      "2012-04-24 22:06:10,072 WARN hdfs.DFSClient (DFSOutputStream.java:run(549)) - DataStreamer Exception",
      "2012-04-24 22:06:10,072 WARN hdfs.DFSClient (DFSOutputStream.java:hflush(1515)) - Error while syncing",
      "2012-05-17 15:39:12,261 ERROR datanode.DataNode (DataXceiver.java:run(193)) - host0.foo.com:49744:DataXceiver error processing TRANSFER_BLOCK operation  src: /127.0.0.1:49811 dest: /127.0.0.1:49744"
    ],
    "stack_traces": [
      "java.io.IOException: Bad connect ack with firstBadLink as *******:50010\nat org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.createBlockOutputStream(DFSOutputStream.java:1053)\nat org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:943)\nat org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:461)",
      "java.io.EOFException: Premature EOF: no length prefix available\nat org.apache.hadoop.hdfs.protocol.HdfsProtoUtil.vintPrefixed(HdfsProtoUtil.java:162)\nat org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.transfer(DFSOutputStream.java:866)\nat org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:843)\nat org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:934)\nat org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:461)",
      "java.io.IOException: BP-2001850558-xx.xx.xx.xx-1337249347060:blk_-8165642083860293107_1002 is neither a RBW nor a Finalized, r=ReplicaBeingWritten, blk_-8165642083860293107_1003, RBW\n getNumBytes()        = 1024\n getBytesOnDisk()  = 1024\n getVisibleLength()= 1024\n getVolume()        = E:\\MyWorkSpace\\branch-2\\Test\\build\\test\\data\\dfs\\data\\data1\\current\n getBlockFile()        = E:\\MyWorkSpace\\branch-2\\Test\\build\\test\\data\\dfs\\data\\data1\\current\\BP-2001850558-xx.xx.xx.xx-1337249347060\\current\\rbw\\blk_-8165642083860293107\n bytesAcked=1024\n bytesOnDisk=102\nat org.apache.hadoop.hdfs.server.datanode.DataNode.transferReplicaForPipelineRecovery(DataNode.java:2038)\nat org.apache.hadoop.hdfs.server.datanode.DataXceiver.transferBlock(DataXceiver.java:525)\nat org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opTransferBlock(Receiver.java:114)\nat org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:78)\nat org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:189)\nat java.lang.Thread.run(Unknown Source)"
    ]
  },
  {
    "file": "HDFS-3441.docx",
    "title": "HDFS-3441",
    "version": "hadoop-2.0.0-alpha",
    "Description": "Race condition between rolling logs at active NN and purging at standby",
    "logs": [
      "2012-05-17 22:15:03,867 INFO org.apache.hadoop.hdfs.server.namenode.FSImage: Image file of size 201 saved in 0 seconds.",
      "2012-05-17 22:15:03,874 INFO org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer: Triggering log roll on remote NameNode /xx.xx.xx.102:8020",
      "2012-05-17 22:15:03,923 INFO org.apache.hadoop.hdfs.server.namenode.NNStorageRetentionManager: Going to retain 2 images with txid >= 111",
      "2012-05-17 22:15:03,923 INFO org.apache.hadoop.hdfs.server.namenode.NNStorageRetentionManager: Purging old image FSImageFile(file=/home/May8/hadoop-3.0.0-SNAPSHOT/hadoop-root/dfs/name/current/fsimage_0000000000000000109, cpktTxId=0000000000000000109)",
      "2012-05-17 22:15:03,961 FATAL org.apache.hadoop.hdfs.server.namenode.FSEditLog: Error: purgeLogsOlderThan 0 failed for required journal (JournalAndStream(mgr=org.apache.hadoop.contrib.bkjournal.BookKeeperJournalManager@142e6767, stream=null))",
      "2012-05-17 22:15:03,963 INFO org.apache.hadoop.hdfs.server.namenode.NameNode: SHUTDOWN_MSG: StanbyNN shuts down."
    ],
    "stack_traces": [
      "java.io.IOException: Exception reading ledger list from zk\nat org.apache.hadoop.contrib.bkjournal.BookKeeperJournalManager.getLedgerList(BookKeeperJournalManager.java:531)\nat org.apache.hadoop.contrib.bkjournal.BookKeeperJournalManager.purgeLogsOlderThan(BookKeeperJournalManager.java:444)\nat org.apache.hadoop.hdfs.server.namenode.JournalSet$5.apply(JournalSet.java:541)\nat org.apache.hadoop.hdfs.server.namenode.JournalSet.mapJournalsAndReportErrors(JournalSet.java:322)\nat org.apache.hadoop.hdfs.server.namenode.JournalSet.purgeLogsOlderThan(JournalSet.java:538)\nat org.apache.hadoop.hdfs.server.namenode.FSEditLog.purgeLogsOlderThan(FSEditLog.java:1011)\nat org.apache.hadoop.hdfs.server.namenode.NNStorageRetentionManager.purgeOldStorage(NNStorageRetentionManager.java:98)\nat org.apache.hadoop.hdfs.server.namenode.FSImage.purgeOldStorage(FSImage.java:900)\nat org.apache.hadoop.hdfs.server.namenode.FSImage.saveFSImageInAllDirs(FSImage.java:885)\nat org.apache.hadoop.hdfs.server.namenode.FSImage.saveNamespace(FSImage.java:822)\nat org.apache.hadoop.hdfs.server.namenode.ha.StandbyCheckpointer.doCheckpoint(StandbyCheckpointer.java:157)\nat org.apache.hadoop.hdfs.server.namenode.ha.StandbyCheckpointer.access$900(StandbyCheckpointer.java:52)\nat org.apache.hadoop.hdfs.server.namenode.ha.StandbyCheckpointer$CheckpointerThread.doWork(StandbyCheckpointer.java:279)\nat org.apache.hadoop.hdfs.server.namenode.ha.StandbyCheckpointer$CheckpointerThread.access$300(StandbyCheckpointer.java:200)\nat org.apache.hadoop.hdfs.server.namenode.ha.StandbyCheckpointer$CheckpointerThread$1.run(StandbyCheckpointer.java:220)\nat org.apache.hadoop.security.SecurityUtil.doAsLoginUserOrFatal(SecurityUtil.java:512)\nat org.apache.hadoop.hdfs.server.namenode.ha.StandbyCheckpointer$CheckpointerThread.run(StandbyCheckpointer.java:216)\nCaused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /nnedits/ledgers/inprogress_72        \nat org.apache.zookeeper.KeeperException.create(KeeperException.java:111)\nat org.apache.zookeeper.KeeperException.create(KeeperException.java:51)\nat org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1113)\nat org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1142)\nat org.apache.hadoop.contrib.bkjournal.EditLogLedgerMetadata.read(EditLogLedgerMetadata.java:113)\nat org.apache.hadoop.contrib.bkjournal.BookKeeperJournalManager.getLedgerList(BookKeeperJournalManager.java:528)\n... 16 more"
    ]
  },
  {
    "file": "HDFS-4201.docx",
    "title": "HDFS-4201",
    "version": "hadoop-2.3.0",
    "Description": "NPE in BPServiceActor#sendHeartBeat",
    "logs": [
      "2012-09-25 04:33:20,782 INFO org.apache.hadoop.hdfs.server.datanode.DataNode: For namenode svsrs00127/11.164.162.226:8020 using DELETEREPORT_INTERVAL of 300000 msec",
      "2012-09-25 04:33:20,782 INFO org.apache.hadoop.hdfs.server.datanode.DataNode: BLOCKREPORT_INTERVAL of 21600000msec Initial delay: 0msec; heartBeatInterval=3000",
      "2012-09-25 04:33:20,782 ERROR org.apache.hadoop.hdfs.server.datanode.DataNode: Exception in BPOfferService for Block pool BP-1678908700-11.164.162.226-1342785481826 (storage id DS-1031100678-11.164.162.251-5010-1341933415989) service to svsrs00127/11.164.162.226:8020"
    ],
    "stack_traces": [
      "java.lang.NullPointerException\nat org.apache.hadoop.hdfs.server.datanode.BPServiceActor.sendHeartBeat(BPServiceActor.java:434)\nat org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:520)\nat org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:673)\nat java.lang.Thread.run(Thread.java:722)"
    ]
  },
  {
    "file": "HDFS-4233.docx",
    "title": "HDFS-4233",
    "version": "hadoop-0.23.5",
    "Description": "NN keeps serving even after no journals started while rolling edit",
    "logs": [
      "2012-xx-yy 00:00:00,000 [IPC Server handler 00 on 0000] INFO org.apache.hadoop.ipc.Server: IPC Server handler 00 on 00000, call: rollEditLog(), rpc version=1, client version=6, methodsFingerPrint=403308677 from 1.1.1.1:12345, error:"
    ],
    "stack_traces": [
      "java.io.IOException: Unable to start log segment 12345678: no journals successfully started.\nat org.apache.hadoop.hdfs.server.namenode.FSEditLog.startLogSegment(FSEditLog.java:840)\nat org.apache.hadoop.hdfs.server.namenode.FSEditLog.rollEditLog(FSEditLog.java:802)\nat org.apache.hadoop.hdfs.server.namenode.FSImage.rollEditLog(FSImage.java:911)\nat org.apache.hadoop.hdfs.server.namenode.FSNamesystem.rollEditLog(FSNamesystem.java:3494)\nat org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.rollEditLog(NameNodeRpcServer.java:644)\nat sun.reflect.GeneratedMethodAccessor111.invoke(Unknown Source)\nat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\nat java.lang.reflect.Method.invoke(Method.java:597)\nat org.apache.hadoop.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:394)\nat org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1530)\nat org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1526)\nat java.security.AccessController.doPrivileged(Native Method)\nat javax.security.auth.Subject.doAs(Subject.java:396)\nat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1212)\nat org.apache.hadoop.ipc.Server$Handler.run(Server.java:1524)"
    ]
  },
  {
    "file": "HDFS-4315.docx",
    "title": "HDFS-4315",
    "version": "hadoop-2.0.2-alpha",
    "Description": "DNs with multiple BPs can have BPOfferServices fail to start due to unsynchronized map access",
    "logs": [
      "2012-12-14 16:30:36,818 WARN  hdfs.DFSClient (DFSOutputStream.java:run(562)) - DataStreamer Exception"
    ],
    "stack_traces": [
      "java.io.IOException: Failed to add a datanode.  User may turn off this feature by setting dfs.client.block.write.replace-datanode-on-failure.policy in configuration, where the current policy is DEFAULT.  (Nodes: current=[127.0.0.1:52552, 127.0.0.1:43557], original=[127.0.0.1:43557, 127.0.0.1:52552])\n  at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:792)\n  at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:852)\n  at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:958)\n  at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:469)",
      "java.lang.NullPointerException\n  at org.apache.hadoop.hdfs.server.datanode.DataNode.initStorage(DataNode.java:850)\n  at org.apache.hadoop.hdfs.server.datanode.DataNode.initBlockPool(DataNode.java:819)\n  at org.apache.hadoop.hdfs.server.datanode.BPOfferService.verifyAndSetNamespaceInfo(BPOfferService.java:308)\n  at org.apache.hadoop.hdfs.server.datanode.BPServiceActor.connectToNNAndHandshake(BPServiceActor.java:218)\n  at org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:660)\n  at java.lang.Thread.run(Thread.java:662)"
    ]
  },
  {
    "file": "HDFS-4404.docx",
    "title": "HDFS-4404",
    "version": "hadoop-2.0.2-alpha",
    "Description": "Create file failure when the machine of first attempted NameNode is down",
    "logs": [
      "2013-01-12 09:51:21,248 DEBUG ipc.Client (Client.java:setupIOstreams(562)) - Connecting to /160.161.0.155:8020",
      "2013-01-12 09:51:38,442 DEBUG ipc.Client (Client.java:close(932)) - closing ipc connection to vm2/160.161.0.155:8020: 10000 millis timeout while waiting for channel to be ready for connect. ch : java.nio.channels.SocketChannel[connection-pending remote=/160.161.0.155:8020]",
      "2013-01-12 09:51:38,443 DEBUG ipc.Client (Client.java:close(940)) - IPC Client (31594013) connection to /160.161.0.155:8020 from hdfs/hadoop@HADOOP.COM: closed",
      "2013-01-12 09:52:47,834 WARN retry.RetryInvocationHandler (RetryInvocationHandler.java:invoke(95)) - Exception while invoking class org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create.",
      "2013-01-12 09:54:52,269 DEBUG ipc.Client (Client.java:stop(1021)) - Stopping client"
    ],
    "stack_traces": [
      "java.net.SocketTimeoutException: 10000 millis timeout while waiting for channel to be ready for connect. ch : java.nio.channels.SocketChannel[connection-pending remote=/160.161.0.155:8020]\nat org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:213)\nat org.apache.hadoop.net.NetUtils.connect(NetUtils.java:524)\nat org.apache.hadoop.net.NetUtils.connect(NetUtils.java:489)\nat org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:474)\nat org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:568)\nat org.apache.hadoop.ipc.Client$Connection.access$2000(Client.java:217)\nat org.apache.hadoop.ipc.Client.getConnection(Client.java:1286)\nat org.apache.hadoop.ipc.Client.call(Client.java:1156)\nat org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:184)\nat $Proxy9.create(Unknown Source)\nat org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:187)\nat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)\nat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\nat java.lang.reflect.Method.invoke(Method.java:597)\nat org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:165)\nat org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:84)\nat $Proxy10.create(Unknown Source)\nat org.apache.hadoop.hdfs.DFSOutputStream.<init>(DFSOutputStream.java:1261)\nat org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:1280)\nat org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1128)\nat org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1086)\nat org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:232)\nat org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:75)\nat org.apache.hadoop.fs.FileSystem.create(FileSystem.java:806)\nat org.apache.hadoop.fs.FileSystem.create(FileSystem.java:787)\nat org.apache.hadoop.fs.FileSystem.create(FileSystem.java:715)\nat test.TestLease.main(TestLease.java:45)",
      "java.net.SocketTimeoutException: Call From szxy1x001833091/172.0.0.13 to vm2:8020 failed on socket timeout exception: java.net.SocketTimeoutException: 10000 millis timeout while waiting for channel to be ready for connect. ch : java.nio.channels.SocketChannel[connection-pending remote=/160.161.0.155:8020]; For more details see:http://wiki.apache.org/hadoop/SocketTimeout\nat org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:743)\nat org.apache.hadoop.ipc.Client.call(Client.java:1180)\nat org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:184)\nat $Proxy9.create(Unknown Source)\nat org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:187)\nat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)\nat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\nat java.lang.reflect.Method.invoke(Method.java:597)\nat org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:165)\nat org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:84)\nat $Proxy10.create(Unknown Source)\nat org.apache.hadoop.hdfs.DFSOutputStream.<init>(DFSOutputStream.java:1261)\nat org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:1280)\nat org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1128)\nat org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1086)\nat org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:232)\nat org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:75)\nat org.apache.hadoop.fs.FileSystem.create(FileSystem.java:806)\nat org.apache.hadoop.fs.FileSystem.create(FileSystem.java:787)\nat org.apache.hadoop.fs.FileSystem.create(FileSystem.java:715)\nat test.TestLease.main(TestLease.java:45)\nCaused by: java.net.SocketTimeoutException: 10000 millis timeout while waiting for channel to be ready for connect. ch : java.nio.channels.SocketChannel[connection-pending remote=/160.161.0.155:8020]\nat org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:213)\nat org.apache.hadoop.net.NetUtils.connect(NetUtils.java:524)\nat org.apache.hadoop.net.NetUtils.connect(NetUtils.java:489)\nat org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:474)\nat org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:568)\nat org.apache.hadoop.ipc.Client$Connection.access$2000(Client.java:217)\nat org.apache.hadoop.ipc.Client.getConnection(Client.java:1286)\nat org.apache.hadoop.ipc.Client.call(Client.java:1156)\n... 20 more"
    ]
  },
  {
    "file": "HDFS-4458 .docx",
    "title": "HDFS-4458",
    "version": "hadoop-2.0.2-alpha",
    "Description": "start balancer failed with \"Failed to create file [/system/balancer.id]\" if configure IP on fs.defaultFS",
    "logs": [
      "2013-01-28 01:21:49,550 WARN org.apache.hadoop.hdfs.StateChange: DIR* NameSystem.startFile: Failed to create file[/system/balancer.id] for [DFSClient_NONMapReduce_413051464_1] on client [10.111.57.222], because this file is already being created by [DFSClient_NONMapReduce_-732728704_1] on [10.111.57.222]",
      "2013-01-28 01:21:49,550 ERROR org.apache.hadoop.security.UserGroupInformation: PriviledgedActionException as:root (auth:SIMPLE) cause:org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException: Failed to create file [/system/balancer.id] for[DFSClient_NONMapReduce_413051464_1] on client [10.111.57.222], because this file is already being created by[DFSClient_NONMapReduce_-732728704_1] on [10.111.57.222]",
      "2013-01-28 01:21:49,551 INFO org.apache.hadoop.ipc.Server: IPC Server handler 2 on 8020, call org.apache.hadoop.hdfs.protocol.ClientProtocol.create from 10.111.57.222:41630: error: org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException: Failed to create file [/system/balancer.id] for[DFSClient_NONMapReduce_413051464_1] on client [10.111.57.222], because this file is already being created by[DFSClient_NONMapReduce_-732728704_1] on [10.111.57.222]",
      "2013-01-28 01:21:49,567 INFO org.apache.hadoop.hdfs.StateChange: BLOCK* NameSystem.allocateBlock: /system/balancer.id. BP-986428099-10.111.57.222-1359353997106 blk_-2102764247427435122_1002",
      "2013-01-28 01:21:49,865 INFO org.apache.hadoop.hdfs.StateChange: BLOCK* addStoredBlock: blockMap updated: 10.111.57.224:50010 is added to blk_-2102764247427435122_1002",
      "2013-01-28 01:21:49,868 INFO org.apache.hadoop.hdfs.StateChange: BLOCK* addStoredBlock: blockMap updated: 10.111.57.225:50010 is added to blk_-2102764247427435122_1002",
      "2013-01-28 01:21:49,871 INFO org.apache.hadoop.hdfs.StateChange: BLOCK* addStoredBlock: blockMap updated: 10.111.57.223:50010 is added to blk_-2102764247427435122_1002",
      "2013-01-28 01:21:49,874 INFO org.apache.hadoop.hdfs.StateChange: DIR* NameSystem.completeFile: file /system/balancer.id is closed by DFSClient_NONMapReduce_-732728704_1"
    ],
    "stack_traces": [
      "org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException: Failed to create file [/system/balancer.id] for[DFSClient_NONMAPREDUCE_413051464_1] on client [10.111.57.222], because this file is already being created by[DFSClient_NONMAPREDUCE_-732728704_1] on [10.111.57.222]\nat org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLeaseInternal(FSNamesystem.java:2175)\nat org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInternal(FSNamesystem.java:1965)\nat org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInt(FSNamesystem.java:1878)\nat org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFile(FSNamesystem.java:1857)\nat org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.create(NameNodeRpcServer.java:407)\nat org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.create(ClientNamenodeProtocolServerSideTranslatorPB.java:208)\nat org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java:45831)\nat org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:454)\nat org.apache.hadoop.ipc.RPC$Server.call(RPC.java:910)\nat org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1694)\nat org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1690)\nat java.security.AccessController.doPrivileged(Native Method)\nat javax.security.auth.Subject.doAs(Subject.java:396)\nat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1367)\nat org.apache.hadoop.ipc.Server$Handler.run(Server.java:1688)"
    ]
  },
  {
    "file": "HDFS-4544.docx",
    "title": "HDFS-4544",
    "version": "hadoop-1.1.1",
    "Description": "Error in deleting blocks should not do check disk, for all types of errors",
    "logs": [
      "2013-03-02 00:08:28,849 WARN org.apache.hadoop.hdfs.server.datanode.DataNode: Unexpected error trying to delete block blk_-2973118207682441648_225738165. BlockInfo not found in volumeMap."
    ],
    "stack_traces": []
  },
  {
    "file": "HDFS-5080.docx",
    "title": "HDFS-5080",
    "version": "hadoop-3.0.0-alpha1",
    "Description": "BootstrapStandby not working with QJM when the existing NN is active",
    "logs": [
      "15/01/22 14:05:06 FATAL ha.BootstrapStandby: Unable to read transaction ids 6175397-6175405 from the configured shared edits storage. Please copy these logs into the shared edits storage or call saveNamespace on the active node.",
      "15/01/22 14:05:06 INFO common.Storage: Storage directory /var/lib/hadoop-hdfs/cache/hdfs/dfs/name has been successfully formatted.",
      "15/01/22 14:05:06 FATAL ha.BootstrapStandby: Unable to read transaction ids 50887593-50995570 from the configured shared edits storage qjournal://us3sm2zk010r07.comp.prod.local:8485;us3sm2zk011r08.comp.prod.local:8485;us3sm2zk012r09.comp.prod.local:8485/whprod. Please copy these logs into the shared edits storage or call saveNamespace on the active node.",
      "15/01/22 14:05:06 INFO util.ExitUtil: Exiting with status 6",
      "15/01/22 14:05:06 INFO namenode.NameNode: SHUTDOWN_MSG:"
    ],
    "stack_traces": [
      "java.io.IOException: Gap in transactions. Expected to be able to read up until at least txid 6175405 but unable to find any edit logs containing txid 6175405\n\tat org.apache.hadoop.hdfs.server.namenode.FSEditLog.checkForGaps(FSEditLog.java:1300)\n\tat org.apache.hadoop.hdfs.server.namenode.FSEditLog.selectInputStreams(FSEditLog.java:1258)\n\tat org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby.checkLogsAvailableForRead(BootstrapStandby.java:229)"
    ]
  },
  {
    "file": "HDFS-5671.docx",
    "title": "HDFS-5671",
    "version": "hadoop-2.2.0",
    "Description": "Fix socket leak in DFSInputStream#getBlockReader",
    "logs": [
      "2013-12-13 15:48:31,474 INFO org.apache.hadoop.hdfs.DFSClient: Will fetch a new access token and retry, access token was invalid when connecting to /192.168.2.27:1004 : org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException: Got access token error for OP_READ_BLOCK, self=/192.168.2.27:56975, remote=/192.168.2.27:1004, for file /hbase/XXXX/b50bf1b95c9242cdd242dc4e6549bc90/raw/d59819ebe5574c79a5d1cf13a733d2ed, for pool BP-621472495-192.168.2.25-1375176775166 block -882505774551713967_11426277",
      "2013-12-13 15:48:31,474 ERROR org.apache.hadoop.hdfs.server.datanode.DataNode: dn2:1004:DataXceiver error processing READ_BLOCK operation src: /192.168.2.27:56975 dest: /192.168.2.27:1004"
    ],
    "stack_traces": [
      "org.apache.hadoop.security.token.SecretManager$InvalidToken: Block token with block_token_identifier (expiryDate=1386914547771, keyId=2020397153, userId=hbase, blockPoolId=BP-621472495-192.168.2.25-1375176775166, blockId=-882505774551713967, access modes=[READ]) is expired.\nat org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.checkAccess(BlockTokenSecretManager.java)\nat org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.checkAccess(BlockTokenSecretManager.java)\nat org.apache.hadoop.hdfs.security.token.block.BlockPoolTokenSecretManager.checkAccess(BlockPoolTokenSecretManager.java)\nat org.apache.hadoop.hdfs.server.datanode.DataXceiver.checkAccess(DataXceiver.java)\nat org.apache.hadoop.hdfs.server.datanode.DataXceiver.readBlock(DataXceiver.java)\nat org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opReadBlock(Receiver.java)\nat org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java)\nat org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java)\nat java.lang.Thread.run(Thread.java)"
    ]
  },
  {
    "file": "HDFS-720.docx",
    "title": "HDFS-720",
    "version": "hadoop-0.21.0",
    "Description": "NPE in BlockReceiver$PacketResponder.run(BlockReceiver.java:923)",
    "logs": [
      "2009-10-21 04:57:02,755 INFO org.apache.hadoop.hdfs.server.datanode.DataNode: Receiving block blk_6345892463926159834_1029 src: /XX,XX,XX.140:37890 dest: /XX.XX.XX.139:51010",
      "2009-10-21 04:57:02,829 INFO org.apache.hadoop.hdfs.server.datanode.DataNode: PacketResponder blk_6345892463926159834_1029 1 Exception java.lang.NullPointerException",
      "2009-10-21 04:57:01,866 INFO org.apache.hadoop.hdfs.server.datanode.DataNode: Receiving block blk_6345892463926159834_1029 src: /XX.XX.XX.140:37385 dest: /XX.XX.XX140:51010",
      "2009-10-21 04:57:02,836 INFO org.apache.hadoop.hdfs.server.datanode.DataNode: PacketResponder 2 for block blk_6345892463926159834_1029 terminating",
      "2009-10-21 04:57:02,885 INFO org.apache.hadoop.hdfs.server.datanode.DataNode: DatanodeRegistration(XX.XX.XX.140:51010, storageID=DS-1292310101-208.76.44.140-51010-1256100924816, infoPort=51075, ipcPort=51020):Exception writing block blk_6345892463926159834_1029 to mirror XX.XX.XX.139:51010          (DN2)",
      "2009-10-21 18:23:11,323 DEBUG org.apache.hadoop.hdfs.server.datanode.DataNode: PacketResponder 1 for block blk_-7356834145770439479_1586 responded my status  for seqno 895",
      "2009-10-21 18:23:11,323 DEBUG org.apache.hadoop.hdfs.server.datanode.DataNode: Receiving one packet for block blk_-7356834145770439479_1586 of length 65024 seqno 896 offsetInBlock 58025472 lastPacketInBlock false",
      "2009-10-21 18:23:11,324 DEBUG org.apache.hadoop.hdfs.server.datanode.DataNode: PacketResponder 1 got seqno = 896",
      "2009-10-21 18:23:11,325 DEBUG org.apache.hadoop.hdfs.server.datanode.DataNode: PacketResponder 1 adding seqno 897 to ack queue.",
      "2009-10-21 18:23:11,325 INFO org.apache.hadoop.hdfs.server.datanode.DataNode: PacketResponder blk_-7356834145770439479_1586 1 Exception ",
      "2009-10-21 18:23:11,326 DEBUG org.apache.hadoop.hdfs.server.datanode.DataNode: PacketResponder 1 got seqno = 4546",
      "2009-10-21 18:23:11,326 DEBUG org.apache.hadoop.hdfs.server.datanode.DataNode: Receiving one packet for block blk_-7356834145770439479_1586 of length 65024 seqno 898 offsetInBlock 58155520 lastPacketInBlock false",
      "2009-10-21 18:23:11,327 INFO org.apache.hadoop.hdfs.server.datanode.DataNode: PacketResponder 1 for block blk_-7356834145770439479_1586 terminating",
      "2009-10-21 18:23:11,338 DEBUG org.apache.hadoop.hdfs.server.datanode.DataNode: PacketResponder 1 for block blk_-830254393316092139_1588 responded my status for seqno 4555",
      "2009-10-21 18:23:11,338 DEBUG org.apache.hadoop.hdfs.server.datanode.DataNode: Number of active connections is: 184",
      "2009-10-21 18:23:11,338 DEBUG org.apache.hadoop.hdfs.server.datanode.DataNode: PacketResponder blk_-830254393316092139_1588 1 responded other status  for seqno 4555",
      "2009-10-21 18:23:11,339 DEBUG org.apache.hadoop.hdfs.server.datanode.DataNode: writeBlock receive buf size 131071 tcp no delay true",
      "2009-10-21 18:23:11,339 INFO org.apache.hadoop.hdfs.server.datanode.DataNode: Receiving block blk_-7356834145770439479_1586 src: /XX,XX,XX.142:49468 dest: /XX,XX,XX.139:51010",
      "2009-10-21 18:23:11,339 INFO org.apache.hadoop.hdfs.server.datanode.DataNode: Recover the RBW replica blk_-7356834145770439479_1586",
      "2009-10-21 18:23:11,339 INFO org.apache.hadoop.hdfs.server.datanode.DataNode: Recovering replica ReplicaBeingWritten, blk_-7356834145770439479_1586, RBW"
    ],
    "stack_traces": [
      "java.lang.NullPointerException\nat org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:923)\nat java.lang.Thread.run(Thread.java:619)",
      "java.io.IOException: Connection reset by peer\nat sun.nio.ch.FileDispatcher.write0(Native Method)\nat sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:29)\nat sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:104)\nat sun.nio.ch.IOUtil.write(IOUtil.java:75)\nat sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:334)\nat org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:55)\nat org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142)\nat org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:146)\nat org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:107)\nat java.io.BufferedOutputStream.write(BufferedOutputStream.java:105)\nat java.io.DataOutputStream.write(DataOutputStream.java:90)\nat org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:466)\nat org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:434)\nat org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:573)\nat org.apache.hadoop.hdfs.server.datanode.DataXceiver.opWriteBlock(DataXceiver.java:352)\nat org.apache.hadoop.hdfs.protocol.DataTransferProtocol$Receiver.opWriteBlock(DataTransferProtocol.java:382)\nat org.apache.hadoop.hdfs.protocol.DataTransferProtocol$Receiver.processOp(DataTransferProtocol.java:323)\nat org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:111)\nat java.lang.Thread.run(Thread.java:619)"
    ]
  },
  {
    "file": "MapReduce-2450.docx",
    "title": "MapReduce-2450",
    "version": "hadoop-0.23.0",
    "Description": "Calls from running tasks to TaskTracker methods sometimes fail and incur a 60s timeout hangs in a corner case",
    "logs": [
      "2009-03-02 21:30:54,384 INFO org.apache.hadoop.metrics.jvm.JvmMetrics: Cannot initialize JVM Metrics with processName=MAP, sessionId= - already initialized",
      "2009-03-02 21:30:54,437 INFO org.apache.hadoop.mapred.MapTask: numReduceTasks: 800",
      "2009-03-02 21:30:54,437 INFO org.apache.hadoop.mapred.MapTask: io.sort.mb = 300",
      "2009-03-02 21:30:55,493 INFO org.apache.hadoop.mapred.MapTask: data buffer = 239075328/298844160",
      "2009-03-02 21:30:55,494 INFO org.apache.hadoop.mapred.MapTask: record buffer = 786432/983040",
      "2009-03-02 21:31:00,381 INFO org.apache.hadoop.mapred.MapTask: Starting flush of map output",
      "2009-03-02 21:31:07,892 INFO org.apache.hadoop.mapred.MapTask: Finished spill 0",
      "2009-03-02 21:31:07,951 INFO org.apache.hadoop.mapred.TaskRunner: Task:attempt_200903022127_0001_m_003163_0 is done. And is in the process of commiting",
      "2009-03-02 21:32:07,949 INFO org.apache.hadoop.mapred.TaskRunner: Communication exception: java.io.IOException: Call to /127.0.0.1:50311 failed on local exception: java.nio.channels.ClosedChannelException",
      "2009-03-02 21:32:07,953 INFO org.apache.hadoop.mapred.TaskRunner: Task 'attempt_200903022127_0001_m_003163_0' done.",
      "2009-03-02 21:31:08,110 WARN org.apache.hadoop.ipc.Server: IPC Server Responder, call ping(attempt_200903022127_0001_m_003163_0) from 127.0.0.1:56884: output error",
      "2009-03-02 21:31:08,111 INFO org.apache.hadoop.ipc.Server: IPC Server handler 10 on 50311 caught: java.nio.channels.ClosedChannelException",
      "2011-04-19 01:10:33,715 INFO org.apache.hadoop.mapred.TaskRunner: Task attempt_201104190103_0002_r_000442_0 is allowed to commit now",
      "2011-04-19 01:10:33,723 INFO org.apache.hadoop.mapred.FileOutputCommitter: Saved output of task 'attempt_201104190103_0002_r_000442_0' to hdfs://r05b02043.yh.aliyun.com:9000/group/dc/willwu/terasort-out",
      "2011-04-19 01:11:33,725 INFO org.apache.hadoop.mapred.TaskRunner: Communication exception: java.io.IOException: Call to /127.0.0.1:48950 failed on local exception: java.nio.channels.ClosedByInterruptException",
      "2011-04-19 01:11:33,728 INFO org.apache.hadoop.mapred.TaskRunner: Task 'attempt_201104190103_0002_r_000442_0' done."
    ],
    "stack_traces": [
      "java.io.IOException: Call to /127.0.0.1:50311 failed on local exception: java.nio.channels.ClosedChannelException\nat org.apache.hadoop.ipc.Client.wrapException(Client.java:765)\nat org.apache.hadoop.ipc.Client.call(Client.java:733)\nat org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:220)\nat org.apache.hadoop.mapred.$Proxy0.ping(Unknown Source)\nat org.apache.hadoop.mapred.Task$TaskReporter.run(Task.java:525)\nat java.lang.Thread.run(Thread.java:619)\nCaused by: java.nio.channels.ClosedChannelException\nat java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:167)\nat java.nio.channels.SelectableChannel.register(SelectableChannel.java:254)\nat org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:331)\nat org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)\nat org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155)\nat org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128)\nat java.io.FilterInputStream.read(FilterInputStream.java:116)\nat org.apache.hadoop.ipc.Client$Connection$PingInputStream.read(Client.java:276)\nat java.io.BufferedInputStream.fill(BufferedInputStream.java:218)\nat java.io.BufferedInputStream.read(BufferedInputStream.java:237)\nat java.io.DataInputStream.readInt(DataInputStream.java:370)\nat org.apache.hadoop.ipc.Client$Connection.receiveResponse(Client.java:501)\nat org.apache.hadoop.ipc.Client$Connection.run(Client.java:446)",
      "java.nio.channels.ClosedChannelException\nat sun.nio.ch.SocketChannelImpl.ensureWriteOpen(SocketChannelImpl.java:126)\nat sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:324)\nat org.apache.hadoop.ipc.Server.channelWrite(Server.java:1195)\nat org.apache.hadoop.ipc.Server.access$1900(Server.java:77)\nat org.apache.hadoop.ipc.Server$Responder.processResponse(Server.java:613)\nat org.apache.hadoop.ipc.Server$Responder.doRespond(Server.java:677)\nat org.apache.hadoop.ipc.Server$Handler.run(Server.java:981)"
    ]
  },
  {
    "file": "MapReduce-2693.docx",
    "title": "MapReduce-2693",
    "version": "hadoop-0.23.0",
    "Description": "NPE in AM causes it to lose containers which are never returned back to RM",
    "logs": [
      "11/06/17 06:11:18 INFO rm.RMContainerAllocator: Assigned based on host match 98.138.163.34",
      "11/06/17 06:11:18 INFO rm.RMContainerRequestor: BEFORE decResourceRequest: applicationId=30 priority=20 resourceName=... numContainers=4978 #asks=5",
      "11/06/17 06:11:18 INFO rm.RMContainerRequestor: AFTER decResourceRequest: applicationId=30 priority=20 resourceName=... numContainers=4977 #asks=5",
      "11/06/17 06:11:18 ERROR rm.RMContainerAllocator: ERROR IN CONTACTING RM."
    ],
    "stack_traces": [
      "java.lang.NullPointerException\nat org.apache.hadoop.mapreduce.v2.app.rm.RMContainerRequestor.decResourceRequest(RMContainerRequestor.java:246)\nat org.apache.hadoop.mapreduce.v2.app.rm.RMContainerRequestor.decContainerReq(RMContainerRequestor.java:198)\nat org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator$ScheduledRequests.assign(RMContainerAllocator.java:523)\nat org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator$ScheduledRequests.access$200(RMContainerAllocator.java:433)\nat org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator.heartbeat(RMContainerAllocator.java:151)\nat org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator$1.run(RMCommunicator.java:220)\nat java.lang.Thread.run(Thread.java:619)"
    ]
  },
  {
    "file": "MapReduce-2953.docx",
    "title": "MapReduce-2953",
    "version": "hadoop-0.23.0",
    "Description": "JobClient fails due to a race in RM, removes staged files and in turn crashes MR AM",
    "logs": [
      "11/09/08 10:52:35 INFO mapreduce.JobSubmitter: number of splits:2094",
      "11/09/08 10:52:36 INFO mapred.YARNRunner: AppMaster capability = memory: 2048,",
      "11/09/08 10:52:36 INFO mapred.YARNRunner: Command to launch container for ApplicationMaster is : $JAVA_HOME/bin/java -Dhadoop.root.logger=INFO,console -Xmx1536m org.apache.hadoop.mapreduce.v2.app.MRAppMaster 1315478927026 1 <FAILCOUNT> 1><LOG_DIR>/stdout 2><LOG_DIR>/stderr",
      "11/09/08 10:52:36 INFO mapred.ResourceMgrDelegate: Submitted application application_1315478927026_1 to ResourceManager",
      "11/09/08 10:52:36 INFO mapreduce.JobSubmitter: Cleaning up the staging area /user/gridperf/.staging/job_1315478927026_0001"
    ],
    "stack_traces": [
      "org.apache.hadoop.yarn.exceptions.impl.pb.YarnRemoteExceptionPBImpl: failed to run job\n\tat org.apache.hadoop.yarn.factories.impl.pb.YarnRemoteExceptionFactoryPBImpl.createYarnRemoteException(YarnRemoteExceptionFactoryPBImpl.java:39)\n\tat org.apache.hadoop.yarn.ipc.RPCUtil.getRemoteException(RPCUtil.java:47)\n\tat org.apache.hadoop.mapred.YARNRunner.submitJob(YARNRunner.java:250)\n\tat org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:377)\n\tat org.apache.hadoop.mapreduce.Job$2.run(Job.java:1072)\n\tat org.apache.hadoop.mapreduce.Job$2.run(Job.java:1069)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat javax.security.auth.Subject.doAs(Subject.java:396)\n\tat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1135)\n\tat org.apache.hadoop.mapreduce.Job.submit(Job.java:1069)\n\tat org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1089)\n\tat org.apache.hadoop.examples.RandomWriter.run(RandomWriter.java:283)\n\tat org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:69)\n\tat org.apache.hadoop.examples.RandomWriter.main(RandomWriter.java:294)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\n\tat java.lang.reflect.Method.invoke(Method.java:597)\n\tat org.apache.hadoop.util.ProgramDriver$ProgramDescription.invoke(ProgramDriver.java:72)\n\tat org.apache.hadoop.util.ProgramDriver.driver(ProgramDriver.java:144)\n\tat org.apache.hadoop.examples.ExampleDriver.main(ExampleDriver.java:68)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\n\tat java.lang.reflect.Method.invoke(Method.java:597)\n\tat org.apache.hadoop.util.RunJar.main(RunJar.java:189)"
    ]
  },
  {
    "file": "MapReduce-2995.docx",
    "title": "MapReduce-2995",
    "version": "hadoop-0.23.0",
    "Description": "MR AM crashes when a container-launch hangs on a faulty NM",
    "logs": [
      "11/09/12 14:11:38 ERROR impl.TaskAttemptImpl: Can't handle this event at current state"
    ],
    "stack_traces": [
      "org.apache.hadoop.yarn.state.InvalidStateTransitonException: Invalid event: TA_CONTAINER_COMPLETED at ASSIGNED\n\tat org.apache.hadoop.yarn.state.StateMachineFactory.doTransition(StateMachineFactory.java:297)\n\tat org.apache.hadoop.yarn.state.StateMachineFactory.access$300(StateMachineFactory.java:39)\n\tat org.apache.hadoop.yarn.state.StateMachineFactory$InternalStateMachine.doTransition(StateMachineFactory.java:439)\n\tat org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl.handle(TaskAttemptImpl.java:903)\n\tat org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl.handle(TaskAttemptImpl.java:127)\n\tat org.apache.hadoop.mapreduce.v2.app.MRAppMaster$TaskAttemptEventDispatcher.handle(MRAppMaster.java:543)\n\tat org.apache.hadoop.mapreduce.v2.app.MRAppMaster$TaskAttemptEventDispatcher.handle(MRAppMaster.java:536)\n\tat org.apache.hadoop.yarn.event.AsyncDispatcher.dispatch(AsyncDispatcher.java:113)\n\tat org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:75)\n\tat java.lang.Thread.run(Thread.java:619)"
    ]
  },
  {
    "file": "MapReduce-2998.docx",
    "title": "MapReduce-2998",
    "version": "hadoop-0.23.0",
    "Description": "Failing to contact Am/History for jobs: java.io.EOFException in DataInputStream",
    "logs": [
      "11/09/12 17:17:50 INFO mapred.YARNRunner: AppMaster capability = memory: 2048,",
      "11/09/12 17:17:51 INFO mapred.YARNRunner: Command to launch container for ApplicationMaster is : $JAVA_HOME/bin/java -Dhadoop.root.logger=DEBUG,console -Xmx1536m org.apache.hadoop.mapreduce.v2.app.MRAppMaster 1315847180566 6 <FAILCOUNT> 1><LOG_DIR>/stdout 2><LOG_DIR>/stderr",
      "11/09/12 17:17:51 INFO mapred.ResourceMgrDelegate: Submitted application application_1315847180566_6 to ResourceManager",
      "11/09/12 17:17:51 INFO mapred.ClientCache: Connecting to HistoryServer at: 0.0.0.0:10020",
      "11/09/12 17:17:51 INFO ipc.YarnRPC: Creating YarnRPC for org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC",
      "11/09/12 17:17:51 INFO mapred.ClientCache: Connected to HistoryServer at: 0.0.0.0:10020",
      "11/09/12 17:17:51 INFO ipc.HadoopYarnRPC: Creating a HadoopYarnProtoRpc proxy for protocol interface org.apache.hadoop.mapreduce.v2.api.MRClientProtocol",
      "11/09/12 17:17:51 INFO mapreduce.Job: Running job: job_1315847180566_0006",
      "11/09/12 17:17:52 INFO mapreduce.Job: map 0% reduce 0%",
      "11/09/12 17:18:00 INFO mapred.ClientServiceDelegate: Tracking Url of JOB is <IP-ADDRESS>:55361",
      "11/09/12 17:18:00 INFO mapred.ClientServiceDelegate: Connecting to <IP-ADDRESS>:43465",
      "11/09/12 17:18:00 INFO ipc.YarnRPC: Creating YarnRPC for org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC",
      "11/09/12 17:18:00 INFO ipc.HadoopYarnRPC: Creating a HadoopYarnProtoRpc proxy for protocol interface org.apache.hadoop.mapreduce.v2.api.MRClientProtocol",
      "11/09/12 17:18:01 INFO mapred.ClientServiceDelegate: Failed to contact AM/History for job job_1315847180566_0006 Will retry..",
      "11/09/12 17:18:01 INFO mapreduce.Job: Job job_1315847180566_0006 failed with state FAILED",
      "11/09/12 17:18:01 INFO mapreduce.Job: Counters: 0",
      "11/09/13 20:13:01 INFO mapreduce.Job: Job job_1315941172309_0006 failed with state FAILED",
      "11/09/13 20:13:01 INFO mapreduce.Job: Counters: 0",
      "11/09/14 23:51:50 WARN conf.Configuration: mapred.used.genericoptionsparser is deprecated.",
      "11/09/15 12:46:51 WARN monitor.ContainersMonitorImpl: Container [pid=22125,containerID=container_1316090748961_0001_01_000001] is running beyond memory-limits. Current usage : 2150408192bytes. Limit : 2147483648bytes. Killing container."
    ],
    "stack_traces": [
      "java.lang.reflect.UndeclaredThrowableException\nat org.apache.hadoop.mapreduce.v2.api.impl.pb.client.MRClientProtocolPBClientImpl.getTaskAttemptCompletionEvents(MRClientProtocolPBClientImpl.java:179)\nat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)\nat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\nat java.lang.reflect.Method.invoke(Method.java:597)\nat org.apache.hadoop.mapred.ClientServiceDelegate.invoke(ClientServiceDelegate.java:237)\nat org.apache.hadoop.mapred.ClientServiceDelegate.getTaskCompletionEvents(ClientServiceDelegate.java:276)\nat org.apache.hadoop.mapred.YARNRunner.getTaskCompletionEvents(YARNRunner.java:547)\nat org.apache.hadoop.mapreduce.Job.getTaskCompletionEvents(Job.java:540)\nat org.apache.hadoop.mapreduce.Job.monitorAndPrintJob(Job.java:1144)\nat org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1092)\nat org.apache.hadoop.examples.WordCount.main(WordCount.java:84)\nat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)\nat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\nat java.lang.reflect.Method.invoke(Method.java:597)\nat org.apache.hadoop.util.ProgramDriver$ProgramDescription.invoke(ProgramDriver.java:72)\nat org.apache.hadoop.util.ProgramDriver.driver(ProgramDriver.java:144)\nat org.apache.hadoop.examples.ExampleDriver.main(ExampleDriver.java:68)\nat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)\nat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\nat java.lang.reflect.Method.invoke(Method.java:597)\nat org.apache.hadoop.util.RunJar.main(RunJar.java:189)\nCaused by: com.google.protobuf.ServiceException: java.io.IOException: Call to /<IP-ADDRESS>:43465 failed on local exception: java.io.EOFException\nat org.apache.hadoop.yarn.ipc.ProtoOverHadoopRpcEngine$Invoker.invoke(ProtoOverHadoopRpcEngine.java:139)\nat $Proxy8.getTaskAttemptCompletionEvents(Unknown Source)\nat org.apache.hadoop.mapreduce.v2.api.impl.pb.client.MRClientProtocolPBClientImpl.getTaskAttemptCompletionEvents(MRClientProtocolPBClientImpl.java:172)\n... 23 more\nCaused by: java.io.IOException: Call to /<IP-ADDRESS>:43465 failed on local exception: java.io.EOFException\nat org.apache.hadoop.ipc.Client.wrapException(Client.java:1119)\nat org.apache.hadoop.ipc.Client.call(Client.java:1087)\nat org.apache.hadoop.yarn.ipc.ProtoOverHadoopRpcEngine$Invoker.invoke(ProtoOverHadoopRpcEngine.java:136)\n... 25 more\nCaused by: java.io.EOFException\nat java.io.DataInputStream.readInt(DataInputStream.java:375)\nat org.apache.hadoop.ipc.Client$Connection.receiveResponse(Client.java:816)\nat org.apache.hadoop.ipc.Client$Connection.run(Client.java:754)",
      "java.lang.reflect.UndeclaredThrowableException\nat org.apache.hadoop.mapreduce.v2.api.impl.pb.client.MRClientProtocolPBClientImpl.getTaskAttemptCompletionEvents(MRClientProtocolPBClientImpl.java:179)\nat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)\nat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\nat java.lang.reflect.Method.invoke(Method.java:597)\nat org.apache.hadoop.mapred.ClientServiceDelegate.invoke(ClientServiceDelegate.java:237)\nat org.apache.hadoop.mapred.ClientServiceDelegate.getTaskCompletionEvents(ClientServiceDelegate.java:276)\nat org.apache.hadoop.mapred.YARNRunner.getTaskCompletionEvents(YARNRunner.java:546)\nat org.apache.hadoop.mapreduce.Job.getTaskCompletionEvents(Job.java:540)\nat org.apache.hadoop.mapreduce.Job.monitorAndPrintJob(Job.java:1144)\nat org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1092)\nat org.apache.hadoop.examples.WordCount.main(WordCount.java:84)\nat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)\nat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\nat java.lang.reflect.Method.invoke(Method.java:597)\nat org.apache.hadoop.util.ProgramDriver$ProgramDescription.invoke(ProgramDriver.java:72)\nat org.apache.hadoop.util.ProgramDriver.driver(ProgramDriver.java:144)\nat org.apache.hadoop.examples.ExampleDriver.main(ExampleDriver.java:68)\nat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)\nat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\nat java.lang.reflect.Method.invoke(Method.java:597)\nat org.apache.hadoop.util.RunJar.main(RunJar.java:189)\nCaused by: com.google.protobuf.ServiceException: java.net.ConnectException: Call to <HOSTNAME>/<IP-ADDRESS>:50862 failed on connection exception: java.net.ConnectException: Connection refused\nat org.apache.hadoop.yarn.ipc.ProtoOverHadoopRpcEngine$Invoker.invoke(ProtoOverHadoopRpcEngine.java:139)\nat $Proxy8.getTaskAttemptCompletionEvents(Unknown Source)\nat org.apache.hadoop.mapreduce.v2.api.impl.pb.client.MRClientProtocolPBClientImpl.getTaskAttemptCompletionEvents(MRClientProtocolPBClientImpl.java:172)\n... 23 more\nCaused by: java.net.ConnectException: Call to <HOSTNAME>/<IP-ADDRESS>:50862 failed on connection exception: java.net.ConnectException: Connection refused\nat org.apache.hadoop.ipc.Client.wrapException(Client.java:1111)\nat org.apache.hadoop.ipc.Client.call(Client.java:1087)\nat org.apache.hadoop.yarn.ipc.ProtoOverHadoopRpcEngine$Invoker.invoke(ProtoOverHadoopRpcEngine.java:136)\n... 25 more\nCaused by: java.net.ConnectException: Connection refused\nat sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)\nat sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)\nat org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)\nat org.apache.hadoop.net.NetUtils.connect(NetUtils.java:376)\nat org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:459)\nat org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:556)\nat org.apache.hadoop.ipc.Client$Connection.access$2000(Client.java:207)\nat org.apache.hadoop.ipc.Client.getConnection(Client.java:1220)\nat org.apache.hadoop.ipc.Client.call(Client.java:1064)\n... 26 more"
    ]
  },
  {
    "file": "MapReduce-3070.docx",
    "title": "MapReduce-3070",
    "version": "hadoop-0.23.0",
    "Description": "NM not able to register with RM after NM restart",
    "logs": [
      "2011-09-23 01:50:46,705 FATAL nodemanager.NodeManager (NodeManager.java:main(204)) - Error starting NodeManager"
    ],
    "stack_traces": [
      "org.apache.hadoop.yarn.YarnException: Failed to Start org.apache.hadoop.yarn.server.nodemanager.NodeManager\n  at org.apache.hadoop.yarn.service.CompositeService.start(CompositeService.java:78)\n  at org.apache.hadoop.yarn.server.nodemanager.NodeManager.start(NodeManager.java:153)\n  at org.apache.hadoop.yarn.server.nodemanager.NodeManager.main(NodeManager.java:202)\nCaused by: org.apache.avro.AvroRuntimeException: org.apache.hadoop.yarn.exceptions.impl.pb.YarnRemoteExceptionPBImpl: Duplicate registration from the node!\n  at org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl.start(NodeStatusUpdaterImpl.java:141)\n  at org.apache.hadoop.yarn.service.CompositeService.start(CompositeService.java:68)\n  ... 2 more\nCaused by: org.apache.hadoop.yarn.exceptions.impl.pb.YarnRemoteExceptionPBImpl: Duplicate registration from the node!\n  at org.apache.hadoop.yarn.ipc.ProtoOverHadoopRpcEngine$Invoker.invoke(ProtoOverHadoopRpcEngine.java:142)\n  at $Proxy13.registerNodeManager(Unknown Source)\n  at org.apache.hadoop.yarn.server.api.impl.pb.client.ResourceTrackerPBClientImpl.registerNodeManager(ResourceTrackerPBClientImpl.java:59)\n  at org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl.registerWithRM(NodeStatusUpdaterImpl.java:175)\n  at org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl.start(NodeStatusUpdaterImpl.java:137)\n  ... 3 more"
    ]
  },
  {
    "file": "MapReduce-3163.docx",
    "title": "MapReduce-3163",
    "version": "hadoop-0.23.0",
    "Description": "JobClient spews errors when killing MR2 job",
    "logs": [
      "11/10/10 20:26:28 WARN conf.Configuration: mapred.used.genericoptionsparser is deprecated. Instead, use mapreduce.client.genericoptionsparser.used",
      "11/10/10 20:26:28 INFO ipc.YarnRPC: Creating YarnRPC for org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC",
      "11/10/10 20:26:28 INFO mapred.ResourceMgrDelegate: Connecting to ResourceManager at c0309.hal.cloudera.com/172.29.81.91:40012",
      "11/10/10 20:26:28 INFO ipc.HadoopYarnRPC: Creating a HadoopYarnProtoRpc proxy for protocol interface org.apache.hadoop.yarn.api.ClientRMProtocol",
      "11/10/10 20:26:28 INFO mapred.ResourceMgrDelegate: Connected to ResourceManager at c0309.hal.cloudera.com/172.29.81.91:40012",
      "11/10/10 20:26:28 INFO mapred.ClientCache: Connecting to HistoryServer at: c0309.hal.cloudera.com:10020",
      "11/10/10 20:26:28 INFO ipc.YarnRPC: Creating YarnRPC for org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC",
      "11/10/10 20:26:28 INFO mapred.ClientCache: Connected to HistoryServer at: c0309.hal.cloudera.com:10020",
      "11/10/10 20:26:28 INFO ipc.HadoopYarnRPC: Creating a HadoopYarnProtoRpc proxy for protocol interface org.apache.hadoop.mapreduce.v2.api.MRClientProtocol",
      "11/10/10 20:26:29 INFO mapred.ClientServiceDelegate: Tracking Url of JOB is c0312.hal.cloudera.com:38448",
      "11/10/10 20:26:29 INFO mapred.ClientServiceDelegate: Connecting to c0312.hal.cloudera.com:44028",
      "11/10/10 20:26:29 INFO ipc.YarnRPC: Creating YarnRPC for org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC",
      "11/10/10 20:26:29 INFO ipc.HadoopYarnRPC: Creating a HadoopYarnProtoRpc proxy for protocol interface org.apache.hadoop.mapreduce.v2.api.MRClientProtocol",
      "11/10/10 20:26:30 INFO mapred.ClientServiceDelegate: Failed to contact AM/History for job job_1318301330793_0001  Will retry..",
      "11/10/10 20:26:30 INFO mapred.ClientServiceDelegate: Application state is completed. FinalApplicationStatus=UNDEFINED. Redirecting to job history server"
    ],
    "stack_traces": [
      "java.lang.reflect.UndeclaredThrowableException\n\tat org.apache.hadoop.mapreduce.v2.api.impl.pb.client.MRClientProtocolPBClientImpl.getJobReport(MRClientProtocolPBClientImpl.java:111)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\n\tat java.lang.reflect.Method.invoke(Method.java:597)\n\tat org.apache.hadoop.mapred.ClientServiceDelegate.invoke(ClientServiceDelegate.java:266)\n\tat org.apache.hadoop.mapred.ClientServiceDelegate.getJobStatus(ClientServiceDelegate.java:341)\n\tat org.apache.hadoop.mapred.YARNRunner.killJob(YARNRunner.java:465)\n\tat org.apache.hadoop.mapreduce.Job.killJob(Job.java:591)\n\tat org.apache.hadoop.mapreduce.tools.CLI.run(CLI.java:258)\n\tat org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:69)\n\tat org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:83)\n\tat org.apache.hadoop.mapred.JobClient.main(JobClient.java:1072)\nCaused by: com.google.protobuf.ServiceException: java.io.IOException: Failed on local exception: java.io.EOFException; Host Details : local host is: \"c0309.hal.cloudera.com/172.29.81.91\"; destination host is: \"\"c0312.hal.cloudera.com\":44028;\n\tat org.apache.hadoop.yarn.ipc.ProtoOverHadoopRpcEngine$Invoker.invoke(ProtoOverHadoopRpcEngine.java:139)\n\tat $Proxy8.getJobReport(Unknown Source)\n\tat org.apache.hadoop.mapreduce.v2.api.impl.pb.client.MRClientProtocolPBClientImpl.getJobReport(MRClientProtocolPBClientImpl.java:104)\n\t... 12 more\nCaused by: java.io.IOException: Failed on local exception: java.io.EOFException; Host Details : local host is: \"c0309.hal.cloudera.com/172.29.81.91\"; destination host is: \"\"c0312.hal.cloudera.com\":44028;\n\tat org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:619)\n\tat org.apache.hadoop.ipc.Client.call(Client.java:1089)\n\tat org.apache.hadoop.yarn.ipc.ProtoOverHadoopRpcEngine$Invoker.invoke(ProtoOverHadoopRpcEngine.java:136)\n\t... 14 more\nCaused by: java.io.EOFException\n\tat java.io.DataInputStream.readInt(DataInputStream.java:375)\n\tat org.apache.hadoop.ipc.Client$Connection.receiveResponse(Client.java:817)\n\tat org.apache.hadoop.ipc.Client$Connection.run(Client.java:755)"
    ]
  },
  {
    "file": "MapReduce-3186.docx",
    "title": "MapReduce-3186",
    "version": "hadoop-0.23.0",
    "Description": "User jobs are getting hanged if the Resource manager process goes down and comes up while job is getting executed.",
    "logs": [
      "2011-10-14 08:59:28 ERROR org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService: AppAttemptId doesnt exist in cache appattempt_1318579738195_0004_000001"
    ],
    "stack_traces": []
  },
  {
    "file": "MapReduce-3226.docx",
    "title": "MapReduce-3226",
    "version": "hadoop-0.23.0",
    "Description": "Few reduce tasks hanging in a gridmix-run",
    "logs": [
      "2011-10-18 10:34:41,006 INFO org.apache.hadoop.mapred.Task: Communication exception: java.io.IOException: Failed on local exception: java.nio.channels.ClosedByInterruptException; Host Details : local host is: \"host.name.com/$IP\"; destination host is: \"\"host.name.com\":48314;"
    ],
    "stack_traces": [
      "java.io.IOException: Failed on local exception: java.nio.channels.ClosedByInterruptException; Host Details : local host is: \"host.name.com/$IP\"; destination host is: \"\"host.name.com\":48314;\n\tat org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:601)\n\tat org.apache.hadoop.ipc.Client.call(Client.java:1089)\n\tat org.apache.hadoop.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:193)\n\tat $Proxy6.statusUpdate(Unknown Source)\n\tat org.apache.hadoop.mapred.Task$TaskReporter.run(Task.java:671)\n\tat java.lang.Thread.run(Thread.java:619)\nCaused by: java.nio.channels.ClosedByInterruptException\n\tat java.nio.channels.spi.AbstractInterruptibleChannel.end(AbstractInterruptibleChannel.java:184)\n\tat sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:341)\n\tat org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:60)\n\tat org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142)\n\tat org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:151)\n\tat org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:112)\n\tat org.apache.hadoop.security.SaslOutputStream.write(SaslOutputStream.java:168)\n\tat java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:65)\n\tat java.io.BufferedOutputStream.flush(BufferedOutputStream.java:123)\n\tat java.io.DataOutputStream.flush(DataOutputStream.java:106)\n\tat org.apache.hadoop.ipc.Client$Connection.sendParam(Client.java:796)\n\tat org.apache.hadoop.ipc.Client.call(Client.java:1066)\n\tat org.apache.hadoop.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:193)\n\tat $Proxy6.getMapCompletionEvents(Unknown Source)\n\tat org.apache.hadoop.mapreduce.task.reduce.EventFetcher.getMapCompletionEvents(EventFetcher.java:99)\n\tat org.apache.hadoop.mapreduce.task.reduce.EventFetcher.run(EventFetcher.java:65)"
    ]
  },
  {
    "file": "MapReduce-3333.docx",
    "title": "MapReduce-3333",
    "version": "hadoop-0.23.0",
    "Description": "MR AM for sort-job going out of memory",
    "logs": [
      "2011-11-02 11:40:36,438 ERROR [ContainerLauncher #258] org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherImpl: Container launch failed for container_1320233407485_0002_01_001434 : java.lang.reflect.UndeclaredThrowableException"
    ],
    "stack_traces": [
      "java.lang.reflect.UndeclaredThrowableException\n\tat org.apache.hadoop.yarn.api.impl.pb.client.ContainerManagerPBClientImpl.startContainer(ContainerManagerPBClientImpl.java:88)\n\tat org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherImpl$EventProcessor.run(ContainerLauncherImpl.java:290)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\n\tat java.lang.Thread.run(Thread.java:619)\nCaused by: com.google.protobuf.ServiceException: java.io.IOException: Failed on local exception: java.io.IOException: Couldn't set up IO streams; Host Details : local host is: \"gsbl91281.blue.ygrid.yahoo.com/98.137.101.189\"; destination host is: \"\"gsbl91525.blue.ygrid.yahoo.com\":45450;\n\tat org.apache.hadoop.yarn.ipc.ProtoOverHadoopRpcEngine$Invoker.invoke(ProtoOverHadoopRpcEngine.java:139)\n\tat $Proxy20.startContainer(Unknown Source)\n\tat org.apache.hadoop.yarn.api.impl.pb.client.ContainerManagerPBClientImpl.startContainer(ContainerManagerPBClientImpl.java:81)\n\t... 4 more\nCaused by: java.io.IOException: Failed on local exception: java.io.IOException: Couldn't set up IO streams; Host Details : local host is: \"gsbl91281.blue.ygrid.yahoo.com/98.137.101.189\"; destination host is: \"\"gsbl91525.blue.ygrid.yahoo.com\":45450;\n\tat org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:655)\n\tat org.apache.hadoop.ipc.Client.call(Client.java:1089)\n\tat org.apache.hadoop.yarn.ipc.ProtoOverHadoopRpcEngine$Invoker.invoke(ProtoOverHadoopRpcEngine.java:136)\n\t... 6 more\nCaused by: java.io.IOException: Couldn't set up IO streams\n\tat org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:621)\n\tat org.apache.hadoop.ipc.Client$Connection.access$2000(Client.java:205)\n\tat org.apache.hadoop.ipc.Client.getConnection(Client.java:1195)\n\tat org.apache.hadoop.ipc.Client.call(Client.java:1065)\n\t... 7 more\nCaused by: java.lang.OutOfMemoryError: unable to create new native thread\n\tat java.lang.Thread.start0(Native Method)\n\tat java.lang.Thread.start(Thread.java:597)\n\tat org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:614)\n\t... 10 more"
    ]
  },
  {
    "file": "MapReduce-3531.docx",
    "title": "MapReduce-3531",
    "version": "hadoop-0.23.1",
    "Description": "Sometimes java.lang.IllegalArgumentException: Invalid key to HMAC computation in NODE_UPDATE also causing RM to stop scheduling",
    "logs": [
      "2011-12-01 11:56:25,200 INFO org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler: nodeUpdate: <NMHost>:48490 clusterResources: memory: 3225600",
      "2011-12-01 11:56:25,202 ERROR org.apache.hadoop.yarn.server.resourcemanager.ResourceManager: Error in handling event type NODE_UPDATE to the scheduler"
    ],
    "stack_traces": [
      "java.lang.IllegalArgumentException: Invalid key to HMAC computation\n\tat org.apache.hadoop.security.token.SecretManager.createPassword(SecretManager.java:141)\n\tat org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager.createPassword(ContainerTokenSecretManager.java:61)\n\tat org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue.createContainer(LeafQueue.java:1108)\n\tat org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue.getContainer(LeafQueue.java:1091)\n\tat org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue.assignContainer(LeafQueue.java:1137)\n\tat org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue.assignNodeLocalContainers(LeafQueue.java:1001)\n\tat org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue.assignContainersOnNode(LeafQueue.java:973)\n\tat org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue.assignContainers(LeafQueue.java:760)\n\tat org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue.assignContainersToChildQueues(ParentQueue.java:583)\n\tat org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue.assignContainers(ParentQueue.java:513)\n\tat org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler.nodeUpdate(CapacityScheduler.java:569)\n\tat org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler.handle(CapacityScheduler.java:611)\n\tat org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler.handle(CapacityScheduler.java:77)\n\tat org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$SchedulerEventDispatcher$EventProcessor.run(ResourceManager.java:294)\n\tat java.lang.Thread.run(Thread.java:619)\nCaused by: java.security.InvalidKeyException: Secret key expected\n\tat com.sun.crypto.provider.HmacCore.a(DashoA13*..)\n\tat com.sun.crypto.provider.HmacSHA1.engineInit(DashoA13*..)\n\tat javax.crypto.Mac.init(DashoA13*..)\n\tat org.apache.hadoop.security.token.SecretManager.createPassword(SecretManager.java:139)\n\t... 14 more"
    ]
  },
  {
    "file": "MapReduce-3656.docx",
    "title": "MapReduce-3656",
    "version": "hadoop-0.23.1",
    "Description": "Sort job on 350 scale is consistently failing with latest MRV2 code",
    "logs": [
      "2012-01-10 14:36:03,779 INFO org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger: USER=<jobuser>    IP=<ip_address_of_RM>        OPERATION=Stop Container Request        TARGET=ContainerManageImpl      RESULT=SUCCESS APPID=application_1326205390797_0002    CONTAINERID=container_1326205390797_0002_01_000001",
      "2012-01-10 14:36:03,780 INFO 2012-01-10 14:36:03,785 INFO org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container: Processing container_1326205390797_0002_01_000001 of type KILL_CONTAINER",
      "2012-01-10 14:33:14,505 INFO org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl: Processing container_1326205390797_0002_01_000001 of type LAUNCHED",
      "2012-01-10 14:33:14,505 INFO org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl: container_1326205390797_0002_01_000001 Container Transitioned from ACQUIRED to RUNNING",
      "2012-01-10 14:36:03,846 INFO org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl: Processing container_1326205390797_0002_01_000001 of type KILL",
      "2012-01-10 14:36:03,846 INFO org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl: container_1326205390797_0002_01_000001 Container Transitioned from RUNNING to KILLED",
      "2012-01-10 14:36:03,846 INFO org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApp: Completed container: container_1326205390797_0002_01_000001 in state: KILLED event:KILL",
      "2012-01-10 14:34:39,289 INFO [AsyncDispatcher event handler] org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl: Processing job_1326205390797_0002 of type JOB_TASK_COMPLETED",
      "2012-01-10 14:34:39,289 INFO [AsyncDispatcher event handler]org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl: Processing attempt_1326205390797_0002_r_000154_0 of type TA_UPDATE",
      "2012-01-10 14:34:39,289 INFO [AsyncDispatcher event handler] org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl: Processing job_1326205390797_0002 of type JOB_COUNTER_UPDATE",
      "2012-01-10 14:34:39,290 ERROR [AsyncDispatcher event handler] org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl: Can't handle this event at current state",
      "2012-01-10 14:36:03,755 INFO [AsyncDispatcher event handler] org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler: Moved tmp to done: hdfs://NN/mapred/history/done_intermediate/jobuser/job_1326205390797_0002.summary_tmp to hdfs://NN/mapred/history/done_intermediate/jobuser/job_1326205390797_0002.summary",
      "2012-01-10 14:36:03,755 INFO [AsyncDispatcher event handler] org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler: Moved tmp to done: hdfs://NN/mapred/history/done_intermediate/jobuser/job_1326205390797_0002_conf.xml_tmp to hdfs://NN/mapred/history/done_intermediate/jobuser/job_1326205390797_0002_conf.xml",
      "2012-01-10 14:36:03,756 INFO [AsyncDispatcher event handler] org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler: Moved tmp to done: hdfs://NN/mapred/history/done_intermediate/jobuser/job_1326205390797_0002-1326205993489-jobuser-sorter-1326206079288-0-0-ERROR-default.jhist_tmp to hdfs://NN/mapred/history/done_intermediate/jobuser/job_1326205390797_0002-1326205993489-jobuser-sorter-1326206079288-0-0-ERROR-default.jhist",
      "2012-01-10 14:36:03,756 INFO [AsyncDispatcher event handler] org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler: Stopped JobHistoryEventHandler. super.stop()",
      "2012-01-10 14:36:03,756 INFO [AsyncDispatcher event handler] org.apache.hadoop.yarn.service.AbstractService: Service:JobHistoryEventHandler is stopped.",
      "2012-01-10 14:36:03,756 ERROR [ContainerLauncher Event Handler] org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherImpl: Returning, interrupted : java.lang.InterruptedException",
      "2012-01-10 14:36:03,759 INFO [AsyncDispatcher event handler] org.apache.hadoop.yarn.service.AbstractService: Service:org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherImpl is stopped.",
      "2012-01-10 14:36:03,759 INFO [AsyncDispatcher event handler] org.apache.hadoop.yarn.service.AbstractService: Service:org.apache.hadoop.mapreduce.v2.app.MRAppMaster$ContainerLauncherRouter is stopped.",
      "2012-01-10 14:36:03,760 INFO [RMCommunicator Allocator] org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator: Allocated thread interrupted. Returning.",
      "2012-01-10 14:36:03,760 ERROR [Thread-27] org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator: Returning,interrupted : java.lang.InterruptedException",
      "2012-01-10 14:36:03,761 INFO [AsyncDispatcher event handler] org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator: Setting job diagnostics to Invalid event TA_UPDATE on TaskAttempt attempt_1326205390797_0002_m_007534_0 Invalid event JOB_COUNTER_UPDATE on Job job_1326205390797_0002 Invalid event JOB_COUNTER_UPDATE on Job job_1326205390797_0002 Invalid event JOB_COUNTER_UPDATE on Job job_1326205390797_0002 Invalid event JOB_COUNTER_UPDATE on Job job_1326205390797_0002 Invalid event JOB_COUNTER_UPDATE on Job job_1326205390797_0002",
      "2012-01-11 13:50:21,402 ERROR org.apache.hadoop.yarn.state.InvalidStateTransitonException",
      "2012-01-11 13:50:21,402 ERROR org.apache.hadoop.yarn.state.InvalidStateTransitonException: Invalid event: TA_UPDATE at ASSIGNED",
      "2012-01-11 13:50:21,402 INFO org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger: USER=<jobuser> IP=<ip_addres_of_RM>         OPERATION=Stop Container Request         TARGET=ContainerManageImpl          RESULT=SUCCESS           APPID=application_1326289061888_0002          CONTAINERID=container_1326289061888_0002_01_000001 (AM)",
      "2012-01-11 13:50:21,406 INFO org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container: Processing container_1326289061888_0002_01_000001 of type KILL_CONTAINER",
      "2012-01-11 13:50:21,407 INFO org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container: Container container_1326289061888_0002_01_000001 transitioned from RUNNING to KILLING",
      "2012-01-11 13:50:21,407 INFO org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch: Cleaning up container container_1326289061888_0002_01_000001",
      "2012-01-11 13:50:21,432 INFO org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container: Processing container_1326289061888_0002_01_000001 of type UPDATE_DIAGNOSTICS_MSG",
      "2012-01-11 13:50:21,449 INFO org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container: Processing container_1326289061888_0002_01_000001 of type CONTAINER_KILLED_ON_REQUEST",
      "2012-01-11 13:47:34,326 INFO org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl: Processing container_1326289061888_0002_01_000001 of type LAUNCHED",
      "2012-01-11 13:47:34,326 INFO org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl: container_1326289061888_0002_01_000001 Container Transitioned from ACQUIRED to RUNNING",
      "2012-01-11 13:50:22,377 INFO org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl: Processing container_1326289061888_0002_01_000001 of type KILL",
      "2012-01-11 13:50:22,377 INFO org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl: container_1326289061888_0002_01_000001 Container Transitioned from RUNNING to KILLED",
      "2012-01-11 13:50:22,377 INFO org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApp: Completed container: container_1326289061888_0002_01_000001 in state: KILLED event:KILL",
      "2012-01-11 13:48:45,526 ERROR [AsyncDispatcher event handler] org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl: Can't handle this event at current state for attempt_1326289061888_0002_m_006598_0"
    ],
    "stack_traces": [
      "org.apache.hadoop.yarn.state.InvalidStateTransitonException: Invalid event: JOB_COUNTER_UPDATE at ERROR\n\tat org.apache.hadoop.yarn.state.StateMachineFactory.doTransition(StateMachineFactory.java:301)\n\tat org.apache.hadoop.yarn.state.StateMachineFactory.access$300(StateMachineFactory.java:43)\n\tat org.apache.hadoop.yarn.state.StateMachineFactory$InternalStateMachine.doTransition(StateMachineFactory.java:443)\n\tat org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl.handle(JobImpl.java:699)\n\tat org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl.handle(JobImpl.java:112)\n\tat org.apache.hadoop.mapreduce.v2.app.MRAppMaster$JobEventDispatcher.handle(MRAppMaster.java:849)\n\tat org.apache.hadoop.mapreduce.v2.app.MRAppMaster$JobEventDispatcher.handle(MRAppMaster.java:845)\n\tat org.apache.hadoop.yarn.event.AsyncDispatcher.dispatch(AsyncDispatcher.java:125)\n\tat org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:82)\n\tat java.lang.Thread.run(Thread.java:619)",
      "org.apache.hadoop.yarn.state.InvalidStateTransitonException: Invalid event: TA_UPDATE at ASSIGNED\n\tat org.apache.hadoop.yarn.state.StateMachineFactory.doTransition(StateMachineFactory.java:301)\n\tat org.apache.hadoop.yarn.state.StateMachineFactory.access$300(StateMachineFactory.java:43)\n\tat org.apache.hadoop.yarn.state.StateMachineFactory$InternalStateMachine.doTransition(StateMachineFactory.java:443)\n\tat org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl.handle(TaskAttemptImpl.java:919)\n\tat org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl.handle(TaskAttemptImpl.java:130)\n\tat org.apache.hadoop.mapreduce.v2.app.MRAppMaster$TaskAttemptEventDispatcher.handle(MRAppMaster.java:871)\n\tat org.apache.hadoop.mapreduce.v2.app.MRAppMaster$TaskAttemptEventDispatcher.handle(MRAppMaster.java:863)\n\tat org.apache.hadoop.yarn.event.AsyncDispatcher.dispatch(AsyncDispatcher.java:125)\n\tat org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:82)\n\tat java.lang.Thread.run(Thread.java:619)"
    ]
  },
  {
    "file": "MapReduce-3714.docx",
    "title": "MapReduce-3714",
    "version": "hadoop-0.23.0",
    "Description": "Reduce hangs in a corner case",
    "logs": [
      "2011-10-18 10:34:41,006 INFO org.apache.hadoop.mapred.Task: Communication exception: java.io.IOException: Failed on local exception: java.nio.channels.ClosedByInterruptException; Host Details : local host is: hostname.com/$ip_addr\"; destination host is: \"\"hostname.com\":12345;"
    ],
    "stack_traces": [
      "java.io.IOException: Failed on local exception: java.nio.channels.ClosedByInterruptException; Host Details : local host is: hostname.com/$ip_addr\"; destination host is: \"\"hostname.com\":12345;\n\tat org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:601)\n\tat org.apache.hadoop.ipc.Client.call(Client.java:1089)\n\tat org.apache.hadoop.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:193)\n\tat $Proxy6.statusUpdate(Unknown Source)  \n\tat org.apache.hadoop.mapred.Task$TaskReporter.run(Task.java:671)\n\tat java.lang.Thread.run(Thread.java:619)\nCaused by: java.nio.channels.ClosedByInterruptException\n\tat java.nio.channels.spi.AbstractInterruptibleChannel.end(AbstractInterruptibleChannel.java:184)\n\tat sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:341)\n\tat org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:60)\n\tat org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142)\n\tat org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:151)\n\tat org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:112)\n\tat org.apache.hadoop.security.SaslOutputStream.write(SaslOutputStream.java:168)\n\tat java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:65)\n\tat java.io.BufferedOutputStream.flush(BufferedOutputStream.java:123)\n\tat java.io.DataOutputStream.flush(DataOutputStream.java:106)\n\tat org.apache.hadoop.ipc.Client$Connection.sendParam(Client.java:796)\n\tat org.apache.hadoop.ipc.Client.call(Client.java:1066)\n\tat org.apache.hadoop.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:193)\n\tat $Proxy6.getMapCompletionEvents(Unknown Source)\n\tat org.apache.hadoop.mapreduce.task.reduce.EventFetcher.getMapCompletionEvents(EventFetcher.java:99)\n\tat org.apache.hadoop.mapreduce.task.reduce.EventFetcher.run(EventFetcher.java:65)",
      "java.lang.Thread.State: TIMED_WAITING (sleeping)\nat java.lang.Thread.sleep(Native Method)\nat org.apache.hadoop.mapreduce.task.reduce.EventFetcher.run(EventFetcher.java:71)",
      "java.lang.Thread.State: WAITING (on object monitor)\n at java.lang.Object.wait(Native Method)\n- waiting on <0xa94b23d8> (a org.apache.hadoop.mapreduce.task.reduce.EventFetcher)\nat java.lang.Thread.join(Thread.java:1143)\n - locked <0xa94b23d8> (a org.apache.hadoop.mapreduce.task.reduce.EventFetcher)\n            at java.lang.Thread.join(Thread.java:1196)\n            at org.apache.hadoop.mapreduce.task.reduce.Shuffle.run(Shuffle.java:135)\n            at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:367)\n            at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:147)\n            at java.security.AccessController.doPrivileged(Native Method)\n            at javax.security.auth.Subject.doAs(Subject.java:396)\n            at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1135)\n            at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:142)"
    ]
  },
  {
    "file": "MapReduce-4074.docx",
    "title": "MapReduce-4074",
    "version": "hadoop-0.23.1",
    "Description": "Client continuously retries to RM When RM goes down before launching Application Master",
    "logs": [
      "28/03/12 07:15:03 INFO ipc.Client: Retrying connect to server: linux-f330.site/10.18.40.182:8032. Already tried 0 time(s).",
      "28/03/12 07:15:12 INFO ipc.Client: Retrying connect to server: linux-f330.site/10.18.40.182:8032. Already tried 9 time(s)."
    ],
    "stack_traces": []
  },
  {
    "file": "MapReduce-4164.docx",
    "title": "MapReduce-4164",
    "version": "hadoop-0.22.0",
    "Description": "Hadoop 22 Exception thrown after task completion causes its reexecution",
    "logs": [
      "2012-02-28 19:17:08,504 INFO org.apache.hadoop.mapred.Merger: Down to the last merge-pass, with 3 segments left of total size: 1969310 bytes",
      "2012-02-28 19:17:08,694 INFO org.apache.hadoop.mapred.Task: Task:attempt_201202272306_0794_m_000094_0 is done. And is in the process of commiting",
      "2012-02-28 19:18:08,774 INFO org.apache.hadoop.mapred.Task: Communication exception: java.io.IOException: Call to /127.0.0.1:35400 failed on local exception: java.nio.channels.ClosedByInterruptException",
      "2012-02-28 19:18:08,825 INFO org.apache.hadoop.mapred.Task: Task 'attempt_201202272306_0794_m_000094_0' done.",
      "2012-02-28 19:17:02,214 INFO org.apache.hadoop.mapred.Merger: Down to the last merge-pass, with 3 segments left of total size: 1974104 bytes",
      "2012-02-28 19:17:02,408 INFO org.apache.hadoop.mapred.Task: Task:attempt_201202272306_0794_m_000000_0 is done. And is in the process of commiting",
      "2012-02-28 19:17:02,519 INFO org.apache.hadoop.mapred.Task: Task 'attempt_201202272306_0794_m_000000_0' done."
    ],
    "stack_traces": [
      "java.io.IOException: Call to /127.0.0.1:35400 failed on local exception: java.nio.channels.ClosedByInterruptException\nat org.apache.hadoop.ipc.Client.wrapException(Client.java:1094)\nat org.apache.hadoop.ipc.Client.call(Client.java:1062)\nat org.apache.hadoop.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:198)\nat $Proxy0.statusUpdate(Unknown Source)\nat org.apache.hadoop.mapred.Task$TaskReporter.run(Task.java:650)\nat java.lang.Thread.run(Thread.java:662)\nCaused by: java.nio.channels.ClosedByInterruptException\nat java.nio.channels.spi.AbstractInterruptibleChannel.end(AbstractInterruptibleChannel.java:184)\nat sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:341)\nat org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:60)\nat org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142)\nat org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:151)\nat org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:112)\nat java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:65)\nat java.io.BufferedOutputStream.flush(BufferedOutputStream.java:123)\nat java.io.DataOutputStream.flush(DataOutputStream.java:106)\nat org.apache.hadoop.ipc.Client$Connection.sendParam(Client.java:769)\nat org.apache.hadoop.ipc.Client.call(Client.java:1040)\n... 4 more"
    ]
  },
  {
    "file": "MapReduce-4448.docx",
    "title": "MapReduce-4448",
    "version": "hadoop-2.0.1-alpha",
    "Description": "Nodemanager crashes upon application cleanup if aggregation failed to start",
    "logs": [
      "[main]2012-07-13 20:35:21,019 INFO org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl: Start request for container_1342210962593_0007_01_000001 by user x",
      "[IPC Server handler 0 on 8041]2012-07-13 20:35:21,043 INFO org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl: Creating a new application reference for app application_1342210962593_0007",
      "[IPC Server handler 0 on 8041]2012-07-13 20:35:21,050 INFO org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application: Application application_1342210962593_0007 transitioned from NEW to INITING",
      "[AsyncDispatcher event handler]2012-07-13 20:35:21,051 INFO org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application: Adding container_1342210962593_0007_01_000001 to application application_1342210962593_0007",
      "[AsyncDispatcher event handler]2012-07-13 20:35:21,062 ERROR org.apache.hadoop.security.UserGroupInformation: PriviledgedActionException as:x (auth:SIMPLE) cause:javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)]",
      "[AsyncDispatcher event handler]2012-07-13 20:35:21,063 WARN org.apache.hadoop.ipc.Client: Exception encountered while connecting to the server : javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)]",
      "[AsyncDispatcher event handler]2012-07-13 20:35:21,063 ERROR org.apache.hadoop.security.UserGroupInformation: PriviledgedActionException as:x (auth:SIMPLE) cause:java.io.IOException: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)]",
      "[AsyncDispatcher event handler]2012-07-13 20:35:21,063 ERROR org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.LogAggregationService: Failed to create user dir [hdfs://xx:8020/mapred/logs/x] while processing app application_1342210962593_0007",
      "[AsyncDispatcher event handler]2012-07-13 20:35:21,064 ERROR org.apache.hadoop.security.UserGroupInformation: PriviledgedActionException as:x (auth:SIMPLE) cause:java.io.IOException: Failed on local exception: java.io.IOException: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)]; Host Details : local host is: \"xx/xx.xx.xx.xx\"; destination host is: \"\"x\":8020;",
      "[AsyncDispatcher event handler]2012-07-13 20:35:21,065 INFO org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application: ✦ application: INITING → FINISHING_CONTAINERS_WAIT; Application application_1342210962593_0007 transitioned from INITING to FINISHING_CONTAINERS_WAIT",
      "[AsyncDispatcher event handler]2012-07-13 20:35:21,067 INFO org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container: Container container_1342210962593_0007_01_000001 transitioned from NEW to DONE",
      "[AsyncDispatcher event handler]2012-07-13 20:35:21,067 INFO org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application: Removing container_1342210962593_0007_01_000001 from application application_1342210962593_0007",
      "[AsyncDispatcher event handler]2012-07-13 20:35:21,069 INFO org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application: Application application_1342210962593_0007 transitioned from FINISHING_CONTAINERS_WAIT to APPLICATION_RESOURCES_CLEANINGUP",
      "[AsyncDispatcher event handler]2012-07-13 20:35:21,070 FATAL org.apache.hadoop.yarn.event.AsyncDispatcher: Error in dispatcher thread",
      "2012-07-13 20:35:21,071 INFO org.apache.hadoop.yarn.event.AsyncDispatcher: Exiting, bbye..",
      "[AsyncDispatcher event handler]2012-07-13 20:35:21,072 WARN org.apache.hadoop.yarn.event.AsyncDispatcher: AsyncDispatcher thread interrupted",
      "2012-07-13 20:35:21,072 INFO org.apache.hadoop.yarn.service.AbstractService: Service:Dispatcher is stopped.",
      "[Thread-1]2012-07-13 20:35:21,073 INFO org.mortbay.log: Stopped SelectChannelConnector@0.0.0.0:8042",
      "[Thread-1]2012-07-13 20:35:21,075 INFO org.apache.hadoop.yarn.service.AbstractService: Service:org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer is stopped.",
      "[Thread-1]2012-07-13 20:35:21,075 INFO org.apache.hadoop.ipc.Server: Stopping server on 8041",
      "[Thread-1]2012-07-13 20:35:21,076 INFO org.apache.hadoop.ipc.Server: Stopping IPC Server listener on 8041",
      "[IPC Server listener on 8041]2012-07-13 20:35:21,077 INFO org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.LogAggregationService: org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.LogAggregationService waiting for pending aggregation during exit",
      "[Thread-1]2012-07-13 20:35:21,077 INFO org.apache.hadoop.ipc.Server: Stopping IPC Server Responder",
      "[IPC Server Responder]2012-07-13 20:35:21,077 INFO org.apache.hadoop.yarn.service.AbstractService: Service:org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.LogAggregationService is stopped."
    ],
    "stack_traces": [
      "org.apache.hadoop.yarn.YarnException: Application is not initialized yet for container_1342210962593_0007_01_000001\n\tat org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.LogAggregationService.stopContainer(LogAggregationService.java:347)\n\tat org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.LogAggregationService.handle(LogAggregationService.java:381)\n\tat org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.LogAggregationService.handle(LogAggregationService.java:65)\n\tat org.apache.hadoop.yarn.event.AsyncDispatcher.dispatch(AsyncDispatcher.java:126)\n\tat org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:75)\n\tat java.lang.Thread.run(Thread.java:619)",
      "java.lang.InterruptedException\n\tat java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:1961)\n\tat java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1996)\n\tat java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)\n\tat org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:69)\n\tat java.lang.Thread.run(Thread.java:619)"
    ]
  },
  {
    "file": "MapReduce-4457.docx",
    "title": "MapReduce-4457",
    "version": "hadoop-0.23.3",
    "Description": "mr job invalid transition TA_TOO_MANY_FETCH_FAILURE at FAILED",
    "logs": [
      "2012-07-16 08:49:53,600 INFO [AsyncDispatcher event handler] org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl: attempt_1342238829791_2501_m_007743_0 TaskAttempt Transitioned from SUCCEEDED to FAILED",
      "2012-07-16 08:49:53,600 INFO [AsyncDispatcher event handler] org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl: attempt_1342238829791_2501_m_008850_0 TaskAttempt Transitioned from SUCCEEDED to FAILED",
      "2012-07-16 08:49:53,600 INFO [AsyncDispatcher event handler] org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl: attempt_1342238829791_2501_m_017344_1000 TaskAttempt Transitioned from RUNNING to SUCCESS_CONTAINER_CLEANUP",
      "2012-07-16 08:49:53,601 ERROR [AsyncDispatcher event handler] org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl: Can't handle this event at current state for attempt_1342238829791_2501_m_000027_0",
      "2012-07-16 08:49:53,601 INFO [AsyncDispatcher event handler] org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl: attempt_1342238829791_2501_m_029091_1000 TaskAttempt Transitioned from RUNNING to SUCCESS_CONTAINER_CLEANUP",
      "2012-07-16 08:49:53,601 INFO [IPC Server handler 17 on 47153] org.apache.hadoop.mapred.TaskAttemptListenerImpl: Status update from attempt_1342238829791_2501_r_000461_1000"
    ],
    "stack_traces": [
      "org.apache.hadoop.yarn.state.InvalidStateTransitonException: Invalid event: TA_TOO_MANY_FETCH_FAILURE at FAILED\nat org.apache.hadoop.yarn.state.StateMachineFactory.doTransition(StateMachineFactory.java:301)\nat org.apache.hadoop.yarn.state.StateMachineFactory.access$300(StateMachineFactory.java:43)\nat org.apache.hadoop.yarn.state.StateMachineFactory$InternalStateMachine.doTransition(StateMachineFactory.java:443)\nat org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl.handle(TaskAttemptImpl.java:954)\nat org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl.handle(TaskAttemptImpl.java:133)\nat org.apache.hadoop.mapreduce.v2.app.MRAppMaster$TaskAttemptEventDispatcher.handle(MRAppMaster.java:913)\nat org.apache.hadoop.mapreduce.v2.app.MRAppMaster$TaskAttemptEventDispatcher.handle(MRAppMaster.java:905)\nat org.apache.hadoop.yarn.event.AsyncDispatcher.dispatch(AsyncDispatcher.java:126)\nat org.apache.hadoop.mapreduce.v2.app.recover.RecoveryService$RecoveryDispatcher.realDispatch(RecoveryService.java:285)\nat org.apache.hadoop.mapreduce.v2.app.recover.RecoveryService$RecoveryDispatcher.dispatch(RecoveryService.java:281)\nat org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:75)\nat java.lang.Thread.run(Thread.java:619)"
    ]
  },
  {
    "file": "MapReduce-4691.docx",
    "title": "MapReduce-4691",
    "version": "hadoop-0.23.3",
    "Description": "Historyserver can report \"Unknown job\" after RM says job has completed",
    "logs": [
      "2012-09-27 20:28:38,068 [main] INFO  org.apache.hadoop.mapred.ClientServiceDelegate - Application state is completed. FinalApplicationStatus=SUCCEEDED. Redirecting to job history server",
      "2012-09-27 20:28:38,530 [main] WARN  org.apache.hadoop.mapred.ClientServiceDelegate - Error from remote end: Unknown job job_1348097917603_3019",
      "2012-09-27 20:28:38,530 [main] ERROR org.apache.hadoop.security.UserGroupInformation - PriviledgedActionException as:xxx (auth:KERBEROS) cause:org.apache.hadoop.yarn.exceptions.impl.pb.YarnRemoteExceptionPBImpl: Unknown job job_1348097917603_3019",
      "2012-09-27 20:28:38,531 [main] WARN  org.apache.pig.tools.pigstats.JobStats - Failed to get map task report"
    ],
    "stack_traces": [
      "org.apache.hadoop.yarn.exceptions.impl.pb.YarnRemoteExceptionPBImpl: Unknown job job_1348097917603_3019\n\tat org.apache.hadoop.yarn.ipc.ProtoOverHadoopRpcEngine$Invoker.invoke(ProtoOverHadoopRpcEngine.java:156)\n\tat $Proxy11.getJobReport(Unknown Source)\n\tat org.apache.hadoop.mapreduce.v2.api.impl.pb.client.MRClientProtocolPBClientImpl.getJobReport(MRClientProtocolPBClientImpl.java:116)\n\tat sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\n\tat java.lang.reflect.Method.invoke(Method.java:597)\n\tat org.apache.hadoop.mapred.ClientServiceDelegate.invoke(ClientServiceDelegate.java:298)\n\tat org.apache.hadoop.mapred.ClientServiceDelegate.getJobStatus(ClientServiceDelegate.java:383)\n\tat org.apache.hadoop.mapred.YARNRunner.getJobStatus(YARNRunner.java:482)\n\tat org.apache.hadoop.mapreduce.Cluster.getJob(Cluster.java:184)"
    ]
  },
  {
    "file": "MapReduce-4801.docx",
    "title": "MapReduce-4801",
    "version": "hadoop-0.23.3",
    "Description": "ShuffleHandler can generate large logs due to prematurely closed channels",
    "logs": [
      "2012-11-15 12:47:02,365 [New I/O server worker #1-14] ERROR org.apache.hadoop.mapred.ShuffleHandler: Shuffle error:",
      "2012-11-15 12:47:02,366 [New I/O server worker #1-14] ERROR org.apache.hadoop.mapred.ShuffleHandler: Shuffle error [id: 0x01901fc1, /xx.xx.xx.xx:xx => /xx.xx.xx.xx:xx] EXCEPTION: java.io.IOException: Connection reset by peer",
      "2012-11-15 12:47:02,366 [New I/O server worker #1-14] ERROR org.apache.hadoop.mapred.ShuffleHandler: Shuffle error:",
      "2012-11-15 12:47:02,367 [New I/O server worker #1-15] ERROR org.apache.hadoop.mapred.ShuffleHandler: Shuffle error:",
      "2012-11-15 12:47:02,367 [New I/O server worker #1-15] ERROR org.apache.hadoop.mapred.ShuffleHandler: Shuffle error [id: 0x01fd225b, /xx.xx.xx.xx:xx => /xx.xx.xx.xx:xx] EXCEPTION: java.io.IOException: Broken pipe",
      "2012-11-15 12:47:02,367 [New I/O server worker #1-15] ERROR org.apache.hadoop.mapred.ShuffleHandler: Shuffle error:"
    ],
    "stack_traces": [
      "java.io.IOException: Connection reset by peer\n\tat sun.nio.ch.FileDispatcher.read0(Native Method)\n\tat sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:21)\n\tat sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:237)\n\tat sun.nio.ch.IOUtil.read(IOUtil.java:204)\n\tat sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:236)\n\tat org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:321)\n\tat org.jboss.netty.channel.socket.nio.NioWorker.processSelectedKeys(NioWorker.java:280)\n\tat org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:200)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\n\tat java.lang.Thread.run(Thread.java:619)",
      "java.nio.channels.ClosedChannelException\n\tat org.jboss.netty.channel.socket.nio.NioWorker.cleanUpWriteBuffer(NioWorker.java:616)\n\tat org.jboss.netty.channel.socket.nio.NioWorker.close(NioWorker.java:592)\n\tat org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:355)\n\tat org.jboss.netty.channel.socket.nio.NioWorker.processSelectedKeys(NioWorker.java:280)\n\tat org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:200)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\n\tat java.lang.Thread.run(Thread.java:619)",
      "java.io.IOException: Broken pipe\n\tat sun.nio.ch.FileDispatcher.write0(Native Method)\n\tat sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:29)\n\tat sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:100)\n\tat sun.nio.ch.IOUtil.write(IOUtil.java:56)\n\tat sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:334)\n\tat org.jboss.netty.channel.socket.nio.SocketSendBufferPool$PooledSendBuffer.transferTo(SocketSendBufferPool.java:239)\n\tat org.jboss.netty.channel.socket.nio.NioWorker.write0(NioWorker.java:469)\n\tat org.jboss.netty.channel.socket.nio.NioWorker.writeFromUserCode(NioWorker.java:387)\n\tat org.jboss.netty.channel.socket.nio.NioServerSocketPipelineSink.handleAcceptedSocket(NioServerSocketPipelineSink.java:137)\n\tat org.jboss.netty.channel.socket.nio.NioServerSocketPipelineSink.eventSunk(NioServerSocketPipelineSink.java:76)\n\tat org.jboss.netty.handler.codec.oneone.OneToOneEncoder.handleDownstream(OneToOneEncoder.java:68)\n\tat org.jboss.netty.handler.stream.ChunkedWriteHandler.flush(ChunkedWriteHandler.java:255)\n\tat org.jboss.netty.handler.stream.ChunkedWriteHandler.handleDownstream(ChunkedWriteHandler.java:124)\n\tat org.jboss.netty.channel.Channels.write(Channels.java:611)\n\tat org.jboss.netty.channel.Channels.write(Channels.java:578)\n\tat org.jboss.netty.channel.AbstractChannel.write(AbstractChannel.java:259)\n\tat org.apache.hadoop.mapred.ShuffleHandler$Shuffle.sendMapOutput(ShuffleHandler.java:477)\n\tat org.apache.hadoop.mapred.ShuffleHandler$Shuffle.messageReceived(ShuffleHandler.java:397)\n\tat org.jboss.netty.handler.stream.ChunkedWriteHandler.handleUpstream(ChunkedWriteHandler.java:148)\n\tat org.jboss.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:116)\n\tat org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:302)\n\tat org.jboss.netty.handler.codec.replay.ReplayingDecoder.unfoldAndfireMessageReceived(ReplayingDecoder.java:522)\n\tat org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:506)\n\tat org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:443)\n\tat org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:274)\n\tat org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:261)\n\tat org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:349)\n\tat org.jboss.netty.channel.socket.nio.NioWorker.processSelectedKeys(NioWorker.java:280)\n\tat org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:200)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\n\tat java.lang.Thread.run(Thread.java:619)",
      "java.nio.channels.ClosedChannelException\n\tat org.jboss.netty.channel.socket.nio.NioWorker.cleanUpWriteBuffer(NioWorker.java:636)\n\tat org.jboss.netty.channel.socket.nio.NioWorker.close(NioWorker.java:592)\n\tat org.jboss.netty.channel.socket.nio.NioWorker.write0(NioWorker.java:512)\n\tat org.jboss.netty.channel.socket.nio.NioWorker.writeFromUserCode(NioWorker.java:387)\n\tat org.jboss.netty.channel.socket.nio.NioServerSocketPipelineSink.handleAcceptedSocket(NioServerSocketPipelineSink.java:137)\n\tat org.jboss.netty.channel.socket.nio.NioServerSocketPipelineSink.eventSunk(NioServerSocketPipelineSink.java:76)\n\tat org.jboss.netty.handler.codec.oneone.OneToOneEncoder.handleDownstream(OneToOneEncoder.java:68)\n\tat org.jboss.netty.handler.stream.ChunkedWriteHandler.flush(ChunkedWriteHandler.java:255)\n\tat org.jboss.netty.handler.stream.ChunkedWriteHandler.handleDownstream(ChunkedWriteHandler.java:124)\n\tat org.jboss.netty.channel.Channels.write(Channels.java:611)\n\tat org.jboss.netty.channel.Channels.write(Channels.java:578)\n\tat org.jboss.netty.channel.AbstractChannel.write(AbstractChannel.java:259)\n\tat org.apache.hadoop.mapred.ShuffleHandler$Shuffle.sendMapOutput(ShuffleHandler.java:477)\n\tat org.apache.hadoop.mapred.ShuffleHandler$Shuffle.messageReceived(ShuffleHandler.java:397)\n\tat org.jboss.netty.handler.stream.ChunkedWriteHandler.handleUpstream(ChunkedWriteHandler.java:148)\n\tat org.jboss.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:116)\n\tat org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:302)\n\tat org.jboss.netty.handler.codec.replay.ReplayingDecoder.unfoldAndfireMessageReceived(ReplayingDecoder.java:522)\n\tat org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:506)\n\tat org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:443)\n\tat org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:274)\n\tat org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:261)\n\tat org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:349)\n\tat org.jboss.netty.channel.socket.nio.NioWorker.processSelectedKeys(NioWorker.java:280)\n\tat org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:200)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)\n\tat java.lang.Thread.run(Thread.java:619)"
    ]
  },
  {
    "file": "MapReduce-4842.docx",
    "title": "MapReduce-4842",
    "version": "hadoop-2.0.2-alpha",
    "Description": "Shuffle race can hang reducer",
    "logs": [
      "2013-07-18 04:32:28,135 INFO [main] org.apache.hadoop.mapreduce.task.reduce.MergeManager: MergerManager: memoryLimit=1503238528, maxSingleShuffleLimit=375809632, mergeThreshold=992137472, ioSortFactor=10, memToMemMergeOutputsThreshold=10",
      "2013-07-18 04:32:28,138 INFO [EventFetcher for fetching Map Completion Events] org.apache.hadoop.mapreduce.task.reduce.EventFetcher: attempt_1373902166027_0622_r_000001_0 Thread started: EventFetcher for fetching Map Completion Events",
      "2013-07-18 04:32:28,146 INFO [EventFetcher for fetching Map Completion Events] org.apache.hadoop.mapreduce.task.reduce.EventFetcher: attempt_1373902166027_0622_r_000001_0: Got 1 new map-outputs",
      "2013-07-18 04:32:28,146 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.ShuffleScheduler: Assiging 101-09-04.sc1.verticloud.com:8080 with 1 to fetcher#1",
      "2013-07-18 04:32:28,146 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.ShuffleScheduler: assigned 1 of 1 to 101-09-04.sc1.verticloud.com:8080 to fetcher#1",
      "2013-07-18 04:32:28,319 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.Fetcher: for url=8080/mapOutput?job=job_1373902166027_0622&reduce=1&map=attempt_1373902166027_0622_m_000017_0 sent hash and receievd reply",
      "2013-07-18 04:32:28,320 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.Fetcher: fetcher#1 about to shuffle output of map attempt_1373902166027_0622_m_000017_0 decomp: 27 len: 31 to MEMORY",
      "2013-07-18 04:32:28,325 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.Fetcher: Read 27 bytes from map-output for attempt_1373902166027_0622_m_000017_0",
      "2013-07-18 04:32:28,325 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.MergeManager: closeInMemoryFile -> map-output of size: 27, inMemoryMapOutputs.size() -> 1, commitMemory -> 0, usedMemory ->27",
      "2013-07-18 04:32:28,325 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.ShuffleScheduler: 101-09-04.sc1.verticloud.com:8080 freed by fetcher#1 in 179s",
      "2013-07-18 04:32:33,158 INFO [EventFetcher for fetching Map Completion Events] org.apache.hadoop.mapreduce.task.reduce.EventFetcher: attempt_1373902166027_0622_r_000001_0: Got 1 new map-outputs",
      "2013-07-18 04:32:33,158 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.ShuffleScheduler: Assiging 101-09-04.sc1.verticloud.com:8080 with 1 to fetcher#1",
      "2013-07-18 04:32:33,158 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.ShuffleScheduler: assigned 1 of 1 to 101-09-04.sc1.verticloud.com:8080 to fetcher#1",
      "2013-07-18 04:32:33,161 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.Fetcher: for url=8080/mapOutput?job=job_1373902166027_0622&reduce=1&map=attempt_1373902166027_0622_m_000016_0 sent hash and receievd reply",
      "2013-07-18 04:32:33,200 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.Fetcher: fetcher#1 about to shuffle output of map attempt_1373902166027_0622_m_000016_0 decomp: 55841282 len: 55841286 to MEMORY",
      "2013-07-18 04:32:33,322 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.Fetcher: Read 55841282 bytes from map-output for attempt_1373902166027_0622_m_000016_0",
      "2013-07-18 04:32:33,323 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.MergeManager: closeInMemoryFile -> map-output of size: 55841282, inMemoryMapOutputs.size() -> 2, commitMemory -> 27, usedMemory ->55841309",
      "2013-07-18 04:32:39,594 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.Fetcher: Read 118022137 bytes from map-output for attempt_1373902166027_0622_m_000015_0",
      "2013-07-18 04:32:39,594 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.MergeManager: closeInMemoryFile -> map-output of size: 118022137, inMemoryMapOutputs.size() -> 3, commitMemory -> 55841309, usedMemory ->173863446",
      "2013-07-18 04:32:39,594 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.ShuffleScheduler: 101-09-04.sc1.verticloud.com:8080 freed by fetcher#1 in 413s",
      "2013-07-18 04:32:42,188 INFO [EventFetcher for fetching Map Completion Events] org.apache.hadoop.mapreduce.task.reduce.EventFetcher: attempt_1373902166027_0622_r_000001_0: Got 1 new map-outputs",
      "2013-07-18 04:32:42,188 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.ShuffleScheduler: Assiging 101-09-04.sc1.verticloud.com:8080 with 1 to fetcher#1",
      "2013-07-18 04:32:42,188 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.ShuffleScheduler: assigned 1 of 1 to 101-09-04.sc1.verticloud.com:8080 to fetcher#1",
      "2013-07-18 04:32:42,190 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.Fetcher: for url=8080/mapOutput?job=job_1373902166027_0622&reduce=1&map=attempt_1373902166027_0622_m_000014_0 sent hash and receievd reply",
      "2013-07-18 04:32:42,277 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.Fetcher: fetcher#1 about to shuffle output of map attempt_1373902166027_0622_m_000014_0 decomp: 140715962 len: 140715966 to MEMORY",
      "2013-07-18 04:32:42,493 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.Fetcher: Read 140715962 bytes from map-output for attempt_1373902166027_0622_m_000014_0",
      "2013-07-18 04:32:42,493 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.MergeManager: closeInMemoryFile -> map-output of size: 140715962, inMemoryMapOutputs.size() -> 4, commitMemory -> 173863446, usedMemory ->314579408",
      "2013-07-18 04:32:42,494 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.ShuffleScheduler: 101-09-04.sc1.verticloud.com:8080 freed by fetcher#1 in 306s",
      "2013-07-18 04:32:43,192 INFO [EventFetcher for fetching Map Completion Events] org.apache.hadoop.mapreduce.task.reduce.EventFetcher: attempt_1373902166027_0622_r_000001_0: Got 1 new map-outputs",
      "2013-07-18 04:32:43,192 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.ShuffleScheduler: Assiging 101-09-04.sc1.verticloud.com:8080 with 1 to fetcher#1",
      "2013-07-18 04:32:43,192 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.ShuffleScheduler: assigned 1 of 1 to 101-09-04.sc1.verticloud.com:8080 to fetcher#1",
      "2013-07-18 04:32:43,195 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.Fetcher: for url=8080/mapOutput?job=job_1373902166027_0622&reduce=1&map=attempt_1373902166027_0622_m_000013_0 sent hash and receievd reply",
      "2013-07-18 04:32:43,280 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.Fetcher: fetcher#1 about to shuffle output of map attempt_1373902166027_0622_m_000013_0 decomp: 141243082 len: 141243086 to MEMORY",
      "2013-07-18 04:32:43,506 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.Fetcher: Read 141243082 bytes from map-output for attempt_1373902166027_0622_m_000013_0",
      "2013-07-18 04:32:43,506 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.MergeManager: closeInMemoryFile -> map-output of size: 141243082, inMemoryMapOutputs.size() -> 5, commitMemory -> 314579408, usedMemory ->455822490",
      "2013-07-18 04:32:43,507 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.ShuffleScheduler: 101-09-04.sc1.verticloud.com:8080 freed by fetcher#1 in 315s",
      "2013-07-18 04:32:44,195 INFO [EventFetcher for fetching Map Completion Events] org.apache.hadoop.mapreduce.task.reduce.EventFetcher: attempt_1373902166027_0622_r_000001_0: Got 1 new map-outputs",
      "2013-07-18 04:32:44,195 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.ShuffleScheduler: Assiging 101-09-04.sc1.verticloud.com:8080 with 1 to fetcher#1",
      "2013-07-18 04:32:44,195 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.ShuffleScheduler: assigned 1 of 1 to 101-09-04.sc1.verticloud.com:8080 to fetcher#1",
      "2013-07-18 04:32:44,198 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.Fetcher: for url=8080/mapOutput?job=job_1373902166027_0622&reduce=1&map=attempt_1373902166027_0622_m_000011_0 sent hash and receievd reply",
      "2013-07-18 04:32:44,305 INFO fetcher#1 org.apache.hadoop.mapreduce.task.reduce.Fetcher: fetcher#1 about to shuffle output of map attempt_1373902166027_0622_m_000011_0 decomp: 173528412 len: 173528416 to MEMORY",
      "2013-07-18 04:32:56,901 INFO fetcher#2 org.apache.hadoop.mapreduce.task.reduce.Fetcher: Read 282474777 bytes from map-output for attempt_1373902166027_0622_m_000001_0",
      "2013-07-18 04:32:56,901 INFO fetcher#2 org.apache.hadoop.mapreduce.task.reduce.MergeManager: closeInMemoryFile -> map-output of size: 282474777, inMemoryMapOutputs.size() -> 5, commitMemory -> 1179552807, usedMemory ->1462027584",
      "2013-07-18 04:32:56,901 INFO fetcher#2 org.apache.hadoop.mapreduce.task.reduce.ShuffleScheduler: 101-09-04.sc1.verticloud.com:8080 freed by fetcher#2 in 2682s",
      "2013-07-18 04:32:56,901 INFO fetcher#4 org.apache.hadoop.mapreduce.task.reduce.ShuffleScheduler: Assiging 101-09-04.sc1.verticloud.com:8080 with 4 to fetcher#4",
      "2013-07-18 04:32:56,902 INFO fetcher#4 org.apache.hadoop.mapreduce.task.reduce.ShuffleScheduler: assigned 4 of 4 to 101-09-04.sc1.verticloud.com:8080 to fetcher#4",
      "2013-07-18 04:32:56,904 INFO fetcher#4 org.apache.hadoop.mapreduce.task.reduce.Fetcher: for url=8080/mapOutput?job=job_1373902166027_0622&reduce=1&map=attempt_1373902166027_0622_m_000006_0,attempt_1373902166027_0622_m_000002_0,attempt_1373902166027_0622_m_000003_0,attempt_1373902166027_0622_m_000005_0 sent hash and receievd reply",
      "2013-07-18 04:32:57,336 INFO [EventFetcher for fetching Map Completion Events] org.apache.hadoop.mapreduce.task.reduce.EventFetcher: attempt_1373902166027_0622_r_000001_0: Got 1 new map-outputs",
      "2013-07-18 04:32:57,414 INFO fetcher#4 org.apache.hadoop.mapreduce.task.reduce.Fetcher: fetcher#4 about to shuffle output of map attempt_1373902166027_0622_m_000006_0 decomp: 280156692 len: 280156696 to MEMORY",
      "2013-07-18 04:32:57,867 INFO fetcher#4 org.apache.hadoop.mapreduce.task.reduce.Fetcher: Read 280156692 bytes from map-output for attempt_1373902166027_0622_m_000006_0",
      "2013-07-18 04:32:57,867 INFO fetcher#4 org.apache.hadoop.mapreduce.task.reduce.MergeManager: closeInMemoryFile -> map-output of size: 280156692, inMemoryMapOutputs.size() -> 6, commitMemory -> 1462027584, usedMemory ->1742184276",
      "2013-07-18 04:32:57,900 INFO fetcher#4 org.apache.hadoop.mapreduce.task.reduce.Fetcher: fetcher#4 - MergerManager returned Status.WAIT ...",
      "2013-07-18 04:32:57,901 INFO fetcher#4 org.apache.hadoop.mapreduce.task.reduce.ShuffleScheduler: 101-09-04.sc1.verticloud.com:8080 freed by fetcher#4 in 999s",
      "2013-07-18 04:32:57,901 INFO fetcher#3 org.apache.hadoop.mapreduce.task.reduce.ShuffleScheduler: Assiging 101-09-04.sc1.verticloud.com:8080 with 4 to fetcher#3",
      "2013-07-18 04:32:57,901 INFO fetcher#3 org.apache.hadoop.mapreduce.task.reduce.ShuffleScheduler: assigned 4 of 4 to 101-09-04.sc1.verticloud.com:8080 to fetcher#3",
      "2013-07-18 04:32:57,903 INFO fetcher#3 org.apache.hadoop.mapreduce.task.reduce.Fetcher: for url=8080/mapOutput?job=job_1373902166027_0622&reduce=1&map=attempt_1373902166027_0622_m_000000_0,attempt_1373902166027_0622_m_000002_0,attempt_1373902166027_0622_m_000005_0,attempt_1373902166027_0622_m_000003_0 sent hash and receievd reply",
      "2013-07-18 04:32:57,904 INFO fetcher#3 org.apache.hadoop.mapreduce.task.reduce.Fetcher: fetcher#3 - MergerManager returned Status.WAIT ..."
    ],
    "stack_traces": []
  },
  {
    "file": "MapReduce-5117.docx",
    "title": "MapReduce-5117",
    "version": "hadoop-2.0.4-alpha",
    "Description": "With security enabled HS delegation token renewer fails",
    "logs": [
      "2013-03-27 23:30:24,680 INFO SecurityLogger.org.apache.hadoop.ipc.Server: Auth successful for oozie/ip-10-46-37-244.ec2.internal@BIGTOP (auth:SIMPLE)",
      "2013-03-27 23:30:24,712 INFO SecurityLogger.org.apache.hadoop.security.authorize.ServiceAuthorizationManager: Authorization successful for rvs (auth:PROXY) via oozie/ip-10-46-37-244.ec2.internal@BIGTOP (auth:KERBEROS) for protocol=interface org.apache.hadoop.yarn.api.ClientRMProtocolPB",
      "2013-03-27 23:30:24,717 INFO org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager: Creating password for identifier: owner=rvs, renewer=yarn, realUser=oozie/ip-10-46-37-244.ec2.internal@BIGTOP, issueDate=1364441424717, maxDate=1365046224717, sequenceNumber=1, masterKeyId=2",
      "2013-03-27 23:30:24,752 INFO org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager: Creating password for identifier: owner=rvs, renewer=yarn, realUser=oozie/ip-10-46-37-244.ec2.internal@BIGTOP, issueDate=1364441424752, maxDate=1365046224752, sequenceNumber=2, masterKeyId=2",
      "2013-03-27 23:30:24,840 INFO org.apache.hadoop.yarn.server.resourcemanager.ClientRMService: Allocated new applicationId: 7",
      "2013-03-27 23:30:26,591 ERROR org.apache.hadoop.ipc.RPC: RPC.stopProxy called on non proxy.",
      "2013-03-27 23:30:26,594 WARN org.apache.hadoop.ipc.Server: IPC Server handler 15 on 8032, call org.apache.hadoop.yarn.api.ClientRMProtocolPB.submitApplication from 10.46.37.244:36960: error: org.apache.hadoop.HadoopIllegalArgumentException: Cannot close proxy - is not Closeable or does not provide closeable invocation handler class org.apache.hadoop.mapreduce.v2.api.impl.pb.client.HSClientProtocolPBClientImpl"
    ],
    "stack_traces": [
      "java.lang.IllegalArgumentException: not a proxy instance\n\tat java.lang.reflect.Proxy.getInvocationHandler(Proxy.java:637)\n\tat org.apache.hadoop.ipc.RPC.stopProxy(RPC.java:609)\n\tat org.apache.hadoop.mapreduce.v2.security.MRDelegationTokenRenewer.stopHistoryProxy(MRDelegationTokenRenewer.java:102)\n\tat org.apache.hadoop.mapreduce.v2.security.MRDelegationTokenRenewer.renew(MRDelegationTokenRenewer.java:71)\n\tat org.apache.hadoop.security.token.Token.renew(Token.java:372)\n\tat org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$1.run(DelegationTokenRenewer.java:370)\n\tat org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$1.run(DelegationTokenRenewer.java:367)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat javax.security.auth.Subject.doAs(Subject.java:396)\n\tat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1478)\n\tat org.apachhadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer.renewToken(DelegationTokenRenewer.java:366)\n\tat org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer.addApplication(DelegationTokenRenewer.java:288)\n\tat org.apache.hadoop.yarn.server.resourcemanager.RMAppManager.submitApplication(RMAppManager.java:269)\n\tat org.apache.hadoop.yarn.server.resourcemanager.RMAppManager.handle(RMAppManager.java:353)\n\tat org.apache.hadoop.yarn.server.resourcemanager.ClientRMService.submitApplication(ClientRMService.java:279)\n\tat org.apache.hadoop.yarn.api.impl.pb.service.ClientRMProtocolPBServiceImpl.submitApplication(ClientRMProtocolPBServiceImpl.java:151)\n\tat org.apache.hadoop.yarn.proto.ClientRMProtocol$ClientRMProtocolService$2.callBlockingMethod(ClientRMProtocol.java:204)\n\tat org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:454)\n\tat org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1014)\n\tat org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1741)\n\tat org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1737)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat javax.security.auth.Subject.doAs(Subject.java:396)\n\tat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1478)\n\tat org.apache.hadoop.ipc.Server$Handler.run(Server.java:1735)",
      "org.apache.hadoop.HadoopIllegalArgumentException: Cannot close proxy - is not Closeable or does not provide closeable invocation handler class org.apache.hadoop.mapreduce.v2.api.impl.pb.client.HSClientProtocolPBClientImpl\n\tat org.apache.hadoop.ipc.RPC.stopProxy(RPC.java:624)\n\tat org.apache.hadoop.mapreduce.v2.security.MRDelegationTokenRenewer.stopHistoryProxy(MRDelegationTokenRenewer.java:102)\n\tat org.apache.hadoop.mapreduce.v2.security.MRDelegationTokenRenewer.renew(MRDelegationTokenRenewer.java:71)\n\tat org.apache.hadoop.security.token.Token.renew(Token.java:372)\n\tat org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$1.run(DelegationTokenRenewer.java:370)\n\tat org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$1.run(DelegationTokenRenewer.java:367)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat javax.security.auth.Subject.doAs(Subject.java:396)\n\tat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1478)\n\tat org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer.renewToken(DelegationTokenRenewer.java:366)\n\tat org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer.addApplication(DelegationTokenRenewer.java:288)\n\tat org.apache.hadoop.yarn.server.resourcemanager.RMAppManager.submitApplication(RMAppManager.java:269)\n\tat org.apache.hadoop.yarn.server.resourcemanager.RMAppManager.handle(RMAppManager.java:353)\n\tat org.apache.hadoop.yarn.server.resourcemanager.ClientRMService.submitApplication(ClientRMService.java:279)\n\tat org.apache.hadoop.yarn.api.impl.pb.service.ClientRMProtocolPBServiceImpl.submitApplication(ClientRMProtocolPBServiceImpl.java:151)\n\tat org.apache.hadoop.yarn.proto.ClientRMProtocol$ClientRMProtocolService$2.callBlockingMethod(ClientRMProtocol.java:204)\n\tat org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:454)\n\tat org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1014)\n\tat org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1741)\n\tat org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1737)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat javax.security.auth.Subject.doAs(Subject.java:396)\n\tat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1478)\n\tat org.apache.hadoop.ipc.Server$Handler.run(Server.java:1735)"
    ]
  },
  {
    "file": "MapReduce-5169.docx",
    "title": "MapReduce-5169",
    "version": "hadoop-1.2.0-rc0",
    "Description": "Job recovery fails if job tracker is restarted after the job is submitted but before its initialized",
    "logs": [
      "13/04/17 22:04:11 INFO mapred.JobClient: Running job: job_201304172049_0008",
      "13/04/17 22:04:12 INFO mapred.JobClient:  map 0% reduce 0%",
      "13/04/17 22:04:32 INFO ipc.Client: Retrying connect to server: host:50300. Already tried 0 time(s); retry policy is MultipleLinearRandomRetry[6x10000ms, 10x60000ms]50300",
      "2013-04-17 22:04:11,305 INFO org.apache.hadoop.mapred.JobInProgress: job_201304172049_0008: nMaps=180 nReduces=1 max=-1",
      "2013-04-17 22:04:11,357 INFO org.apache.hadoop.mapred.JobQueuesManager: Job job_201304172049_0008 submitted to queue default",
      "2013-04-17 22:04:11,357 INFO org.apache.hadoop.mapred.JobTracker: Job job_201304172049_0008 added successfully for user 'hrt_qa' to queue 'default'  JobStatus addJob(JobID jobId, JobInProgress job):  Add a job to the jobtracker. This is the main logic of job submission.",
      "2013-04-17 22:04:14,268 INFO org.apache.hadoop.mapred.JobInitializationPoller: Passing to Initializer Job Id :job_201304172049_0008 User: user Queue : default   Print log statements about which jobs are being passed to init-threads.",
      "2013-04-17 22:04:15,181 INFO org.apache.hadoop.mapred.JobTracker: SHUTDOWN_MSG:",
      "2013-04-17 22:04:15,204 INFO org.apache.hadoop.mapred.JobInitializationPoller: Initializing job : job_201304172049_0008 in Queue default For user : user",
      "2013-04-17 22:04:15,204 INFO org.apache.hadoop.mapred.JobTracker: Initializing job_201304172049_0008",
      "2013-04-17 22:04:15,204 INFO org.apache.hadoop.mapred.JobInProgress: Initializing job_201304172049_0008",
      "2013-04-17 22:04:30,900 INFO org.apache.hadoop.mapred.JobTracker: STARTUP_MSG:",
      "2013-04-17 22:04:31,862 WARN org.apache.hadoop.mapred.JobTracker: Job job_201304172049_0008 does not have valid info/token file so ignoring for recovery"
    ],
    "stack_traces": [
      "java.io.IOException: The job appears to have been removed.\n\tat org.apache.hadoop.mapred.JobClient$NetworkedJob.updateStatus(JobClient.java:241)\n\tat org.apache.hadoop.mapred.JobClient$NetworkedJob.isComplete(JobClient.java:321)\n\tat org.apache.hadoop.mapred.JobClient.monitorAndPrintJob(JobClient.java:1382)\n\tat org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:583)\n\tat org.apache.hadoop.examples.WordCount.main(WordCount.java:82)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\n\tat java.lang.reflect.Method.invoke(Method.java:597)\n\tat org.apache.hadoop.util.ProgramDriver$ProgramDescription.invoke(ProgramDriver.java:68)\n\tat org.apache.hadoop.util.ProgramDriver.driver(ProgramDriver.java:139)\n\tat org.apache.hadoop.examples.ExampleDriver.main(ExampleDriver.java:64)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\n\tat java.lang.reflect.Method.invoke(Method.java:597)\n\tat org.apache.hadoop.util.RunJar.main(RunJar.java:160)"
    ]
  },
  {
    "file": "MapReduce-5466.docx",
    "title": "MapReduce-5466",
    "version": "hadoop-2.1.1-beta",
    "Description": "Historyserver does not refresh the result of restarted jobs after RM restart",
    "logs": [
      "13/08/08 01:24:13 INFO mapred.ClientServiceDelegate: Application state is completed. FinalApplicationStatus=SUCCEEDED. Redirecting to job history server"
    ],
    "stack_traces": []
  },
  {
    "file": "MapReduce-5488.docx",
    "title": "MapReduce-5488",
    "version": "hadoop-2.1.0-beta",
    "Description": "Job recovery fails after killing all the running containers for the app",
    "logs": [
      "13/08/30 08:45:42 INFO impl.YarnClientImpl: Submitted application application_1377851032086_0003 to ResourceManager at hostname/68.142.247.148:8032",
      "13/08/30 08:45:42 INFO mapreduce.Job: Running job: job_1377851032086_0003",
      "13/08/30 08:45:48 INFO mapreduce.Job: Job job_1377851032086_0003 running in uber mode : false",
      "13/08/30 08:45:48 INFO mapreduce.Job:  map 0% reduce 0%",
      "13/08/30 08:45:55 INFO ipc.Client: Retrying connect to server: hostname/68.142.247.155:52713. Already tried 0 time(s); retry policy is RetryUpToMaximumCountWithFixedSleep(maxRetries=1, sleepTime=1 SECONDS) (still try to connect the old AM addr)",
      "13/08/30 08:45:56 INFO ipc.Client: Retrying connect to server: hostname/68.142.247.155:52713. Already tried 0 time(s); retry policy is RetryUpToMaximumCountWithFixedSleep(maxRetries=1, sleepTime=1 SECONDS)",
      "13/08/30 08:45:56 ERROR security.UserGroupInformation: PriviledgedActionException as:user@REALM (auth:KERBEROS) cause:java.io.IOException: java.net.ConnectException: Call From hostname.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused"
    ],
    "stack_traces": [
      "java.io.IOException: java.net.ConnectException: Call From hostname.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused\n\tat org.apache.hadoop.mapred.ClientServiceDelegate.invoke(ClientServiceDelegate.java:319)\n\tat org.apache.hadoop.mapred.ClientServiceDelegate.getTaskCompletionEvents(ClientServiceDelegate.java:354)\n\tat org.apache.hadoop.mapred.YARNRunner.getTaskCompletionEvents(YARNRunner.java:529)\n\tat org.apache.hadoop.mapreduce.Job$5.run(Job.java:668)\n\tat org.apache.hadoop.mapreduce.Job$5.run(Job.java:665)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat javax.security.auth.Subject.doAs(Subject.java:396)\n\tat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1477)\n\tat org.apache.hadoop.mapreduce.Job.getTaskCompletionEvents(Job.java:665)\n\tat org.apache.hadoop.mapreduce.Job.monitorAndPrintJob(Job.java:1349)\n\tat org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1289)\n\tat org.apache.hadoop.examples.WordCount.main(WordCount.java:84)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\n\tat java.lang.reflect.Method.invoke(Method.java:597)\n\tat org.apache.hadoop.util.ProgramDriver$ProgramDescription.invoke(ProgramDriver.java:72)\n\tat org.apache.hadoop.util.ProgramDriver.run(ProgramDriver.java:144)\n\tat org.apache.hadoop.examples.ExampleDriver.main(ExampleDriver.java:74)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)\n\tat java.lang.reflect.Method.invoke(Method.java:597)\n\tat org.apache.hadoop.util.RunJar.main(RunJar.java:212)\nCaused by: java.net.ConnectException: Call From hostname.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused\n\tat org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:783)\n\tat org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:730)\n\tat org.apache.hadoop.ipc.Client.call(Client.java:1351)\n\tat org.apache.hadoop.ipc.Client.call(Client.java:1300)\n\tat org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206)\n\tat $Proxy14.getTaskAttemptCompletionEvents(Unknown Source)\n\tat org.apache.hadoop.mapreduce.v2.api.impl.pb.client.MRClientProtocolPBClientImpl.getTaskAttemptCompletionEvents(MRClientProtocolPBClientImpl.java:177)\n\tat java.lang.reflect.Method.invoke(Method.java:597)\n\tat org.apache.hadoop.mapred.ClientServiceDelegate.invoke(ClientServiceDelegate.java:310)\n\t... 23 more\nCaused by: java.net.ConnectException: Connection refused\n\tat org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:547)\n\tat org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:642)\n\tat org.apache.hadoop.ipc.Client$Connection.access$2600(Client.java:314)\n\tat org.apache.hadoop.ipc.Client.getConnection(Client.java:1399)\n\tat org.apache.hadoop.ipc.Client.call(Client.java:1318)\n\t... 32 more"
    ]
  },
  {
    "file": "MapReduce-5512.docx",
    "title": "MapReduce-5512",
    "version": "hadoop-1.2.0",
    "Description": "TaskTracker hung after failed reconnect to the JobTracker",
    "logs": [
      "2013-09-08 04:20:48,441 ERROR org.apache.hadoop.mapred.TaskTracker: Caught exception: java.io.IOException: Call to jobtrackerhost/100.70.178.36:9010 failed on local exception: java.io.IOException: An existing connection was forcibly closed by the remote host",
      "2013-09-08 04:20:48,441 INFO org.apache.hadoop.mapred.TaskTracker: Resending 'status' to 'jobtrackerhost' with reponseId ‘-27775’ (if the last heartbeat got through, resend the previous status information)",
      "2013-09-08 04:21:09,454 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: jobtrackerhost/100.70.178.36:9010. Already tried 0 time(s); maxRetries=45",
      "2013-09-08 04:22:12,490 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: jobtrackerhost/100.70.178.36:9010. Already tried 3 time(s); maxRetries=45",
      "2013-09-08 04:22:21,723 INFO org.apache.hadoop.mapred.TaskTracker: Recieved ReinitTrackerAction from JobTracker",
      "2013-09-08 04:22:21,801 INFO org.apache.hadoop.filecache.TrackerDistributedCacheManager: Cleanup...",
      "2013-09-08 04:22:21,801 INFO org.apache.hadoop.mapred.TaskTracker: Shutting down: Map-events fetcher for all reduce tasks on tracker_workernode0:127.0.0.1/127.0.0.1:63692",
      "2013-09-08 04:22:21,801 INFO org.apache.hadoop.ipc.Server: Stopping server on 63692",
      "2013-09-08 04:22:21,801 WARN org.apache.hadoop.mapred.TaskTracker: Reinitializing local state",
      "2013-09-08 04:22:21,833 INFO org.apache.hadoop.mapred.TaskTracker: Starting tasktracker with owner as hdp",
      "2013-09-08 04:22:21,833 INFO org.apache.hadoop.mapred.TaskTracker: Good mapred local directories are: c:\\hdfs\\mapred\\local",
      "2013-09-08 04:22:21,833 INFO org.apache.hadoop.ipc.Server: IPC Server handler 0 on 63692: exiting",
      "2013-09-08 04:22:21,833 INFO org.apache.hadoop.ipc.Server: IPC Server handler 2 on 63692: exiting",
      "2013-09-08 04:22:21,864 INFO org.apache.hadoop.mapred.TaskTracker: TaskTracker up at: 127.0.0.1/127.0.0.1:62593",
      "2013-09-08 04:22:21,864 INFO org.apache.hadoop.mapred.TaskTracker: Starting tracker tracker_workernode0:127.0.0.1/127.0.0.1:62593",
      "2013-09-08 04:22:21,864 INFO org.apache.hadoop.ipc.Server: IPC Server Responder: starting",
      "2013-09-08 04:22:22,208 ERROR org.apache.hadoop.security.UserGroupInformation: PriviledgedActionException as:hdp cause:java.io.IOException: Call to jobtrackerhost/100.70.178.36:9010 failed on local exception: java.io.IOException: An existing connection was forcibly closed by the remote host",
      "2013-09-08 04:22:22,223 ERROR org.apache.hadoop.mapred.TaskTracker: Got fatal exception while reinitializing TaskTracker: java.io.IOException: Call to jobtrackerhost/100.70.178.36:9010 failed on local exception: java.io.IOException: An existing connection was forcibly closed by the remote host"
    ],
    "stack_traces": [
      "java.io.IOException: Call to jobtrackerhost/100.70.178.36:9010 failed on local exception: java.io.IOException: An existing connection was forcibly closed by the remote host\n\tat org.apache.hadoop.ipc.Client.wrapException(Client.java:1155)\n\tat org.apache.hadoop.ipc.Client.call(Client.java:1123)\n\tat org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:229)\n\tat org.apache.hadoop.mapred.$Proxy5.heartbeat(Unknown Source)\n\tat org.apache.hadoop.mapred.TaskTracker.transmitHeartBeat(TaskTracker.java:2038)\n\tat org.apache.hadoop.mapred.TaskTracker.offerService(TaskTracker.java:1832)\n\tat org.apache.hadoop.mapred.TaskTracker.run(TaskTracker.java:2685)\n\tat org.apache.hadoop.mapred.TaskTracker.main(TaskTracker.java:3950)\nCaused by: java.io.IOException: An existing connection was forcibly closed by the remote host\n\tat sun.nio.ch.SocketDispatcher.read0(Native Method)\n\tat sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:43)\n\tat sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:225)\n\tat sun.nio.ch.IOUtil.read(IOUtil.java:198)\n\tat sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:359)\n\tat org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:55)\n\tat org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142)\n\tat org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155)\n\tat org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128)\n\tat java.io.FilterInputStream.read(FilterInputStream.java:133)\n\tat org.apache.hadoop.ipc.Client$Connection$PingInputStream.read(Client.java:370)\n\tat java.io.BufferedInputStream.fill(BufferedInputStream.java:235)\n\tat java.io.BufferedInputStream.read(BufferedInputStream.java:254)\n\tat java.io.DataInputStream.readInt(DataInputStream.java:387)\n\tat org.apache.hadoop.ipc.Client$Connection.receiveResponse(Client.java:852)\n\tat org.apache.hadoop.ipc.Client$Connection.run(Client.java:797)",
      "java.lang.InterruptedException: sleep interrupted\n\tat java.lang.Thread.sleep(Native Method)\n\tat org.apache.hadoop.filecache.TrackerDistributedCacheManager$CleanupThread.run(TrackerDistributedCacheManager.java:1038)",
      "java.io.IOException: Call to jobtrackerhost/100.70.178.36:9010 failed on local exception: java.io.IOException: An existing connection was forcibly closed by the remote host\n\tat org.apache.hadoop.ipc.Client.wrapException(Client.java:1155)\n\tat org.apache.hadoop.ipc.Client.call(Client.java:1123)\n\tat org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:229)\n\tat org.apache.hadoop.mapred.$Proxy5.getProtocolVersion(Unknown Source)\n\tat org.apache.hadoop.ipc.RPC.checkVersion(RPC.java:422)\n\tat org.apache.hadoop.ipc.RPC.getProxy(RPC.java:414)\n\tat org.apache.hadoop.ipc.RPC.getProxy(RPC.java:392)\n\tat org.apache.hadoop.ipc.RPC.getProxy(RPC.java:374)\n\tat org.apache.hadoop.ipc.RPC.getProxy(RPC.java:453)\n\tat org.apache.hadoop.ipc.RPC.waitForProxy(RPC.java:335)\n\tat org.apache.hadoop.ipc.RPC.waitForProxy(RPC.java:300)\n\tat org.apache.hadoop.mapred.TaskTracker$3.run(TaskTracker.java:916)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat javax.security.auth.Subject.doAs(Subject.java:415)\n\tat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1233)\n\tat org.apache.hadoop.mapred.TaskTracker.initialize(TaskTracker.java:912)\n\tat org.apache.hadoop.mapred.TaskTracker.run(TaskTracker.java:2713)\n\tat org.apache.hadoop.mapred.TaskTracker.main(TaskTracker.java:3950)\nCaused by: java.io.IOException: An existing connection was forcibly closed by the remote host\n\tat sun.nio.ch.SocketDispatcher.read0(Native Method)\n\tat sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:43)\n\tat sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:225)\n\tat sun.nio.ch.IOUtil.read(IOUtil.java:198)\n\tat sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:359)\n\tat org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:55)\n\tat org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142)\n\tat org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155)\n\tat org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128)\n\tat java.io.FilterInputStream.read(FilterInputStream.java:133)\n\tat org.apache.hadoop.ipc.Client$Connection$PingInputStream.read(Client.java:370)\n\tat java.io.BufferedInputStream.fill(BufferedInputStream.java:235)\n\tat java.io.BufferedInputStream.read(BufferedInputStream.java:254)\n\tat java.io.DataInputStream.readInt(DataInputStream.java:387)\n\tat org.apache.hadoop.ipc.Client$Connection.receiveResponse(Client.java:852)\n\tat org.apache.hadoop.ipc.Client$Connection.run(Client.java:797)"
    ]
  },
  {
    "file": "MapReduce-5542.docx",
    "title": "MapReduce-5542",
    "version": "hadoop-2.1.0-beta",
    "Description": "Killing a job just as it finishes can generate an NPE in client",
    "logs": [
      "13/09/26 14:38:46 INFO client.RMProxy: Connecting to ResourceManager at xx/xx:xx",
      "13/09/26 14:38:48 INFO mapred.ClientServiceDelegate: Application state is completed. FinalApplicationStatus=KILLED. Redirecting to job history server"
    ],
    "stack_traces": [
      "java.lang.NullPointerException\n\tat org.apache.hadoop.mapred.YARNRunner.killJob(YARNRunner.java:563)\n\tat org.apache.hadoop.mapreduce.Job.killJob(Job.java:624)\n\tat org.apache.hadoop.mapreduce.tools.CLI.run(CLI.java:299)\n\tat org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)\n\tat org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:84)\n\tat org.apache.hadoop.mapred.JobClient.main(JobClient.java:1231)"
    ]
  },
  {
    "file": "ZooKeeper-1294.docx",
    "title": "ZooKeeper-1294",
    "version": "zookeeper-3.5.0",
    "Description": "One of the zookeeper server is not accepting any requests",
    "logs": [
      "2011-11-08 15:49:51,360 - WARN [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:65170:NIOServerCnxn@642] - Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running"
    ],
    "stack_traces": []
  },
  {
    "file": "ZooKeeper-1382.docx",
    "title": "ZooKeeper-1382",
    "version": "zookeeper-3.4.5",
    "Description": "Zookeeper server holds onto dead/expired session ids in the watch data structures",
    "logs": [
      "application.log.2012-01-26-325.gz:2012/01/26 04:56:36.177 INFO [ClientCnxn] [main-SendThread(223.prod:12913)] [application] Session establishment complete on server 223.prod/172.17.135.38:12913, sessionid = 0x134485fd7bcb26f, negotiated timeout = 6000",
      "application.log.2012-01-27.gz:2012/01/27 09:52:37.714 INFO [ClientCnxn] [main-SendThread(223.prod:12913)] [application] Client session timed out, have not heard from server in 9827ms for sessionid 0x134485fd7bcb26f, closing socket connection and attempting reconnect",
      "application.log.2012-01-27.gz:2012/01/27 09:52:38.191 INFO [ClientCnxn] [main-SendThread(226.prod:12913)] [application] Unable to reconnect to ZooKeeper service, session 0x134485fd7bcb26f has expired, closing socket connection",
      "zookeeper.log.2012-01-27-leader-225.gz:2012-01-27 09:52:34,010 - INFO [SessionTracker:ZooKeeperServer@314] - Expiring session 0x134485fd7bcb26f, timeout of 6000ms exceeded",
      "zookeeper.log.2012-01-27-leader-225.gz:2012-01-27 09:52:34,010 - INFO [ProcessThread:-1:PrepRequestProcessor@391] - Processed session termination for sessionid: 0x134485fd7bcb26f",
      "zookeeper.log.2012-01-26-223.gz:2012-01-26 04:56:36,173 - INFO [CommitProcessor:1:NIOServerCnxn@1580] - Established session 0x134485fd7bcb26f with negotiated timeout 6000 for client /172.17.136.82:45020",
      "zookeeper.log.2012-01-27-223.gz:2012-01-27 09:52:34,018 - INFO [CommitProcessor:1:NIOServerCnxn@1435] - Closed socket connection for client /172.17.136.82:45020 which had sessionid 0x134485fd7bcb26f",
      "2012-01-27 09:52:38,190 - INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:12913:NIOServerCnxn@770] - Client attempting to renew session 0x134485fd7bcb26f at /172.17.136.82:49367",
      "2012-01-27 09:52:38,191 - INFO [QuorumPeer:/0.0.0.0:12913:NIOServerCnxn@1573] - Invalid session 0x134485fd7bcb26f for client /172.17.136.82:49367, probably expired",
      "2012-01-27 09:52:38,191 - INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:12913:NIOServerCnxn@1435] - Closed socket connection for client /172.17.136.82:49367 which had sessionid 0x134485fd7bcb26f"
    ],
    "stack_traces": []
  },
  {
    "file": "ZooKeeper-1496.docx",
    "title": "ZooKeeper-1496",
    "version": "zookeeper-3.4.3",
    "Description": "Ephemeral node not getting cleared even after client has exited",
    "logs": [
      "2015-02-24 18:01:05,616 INFO org.apache.zookeeper.server.PrepRequestProcessor: Got user-level KeeperException when processing sessionid:0x34bb91dede303c9 type:create cxid:0x1 zxid:0xe00002ce3 txntype:-1 reqpath:n/a Error Path:/hadoop-ha/mynameservice/ActiveStandbyElectorLock Error:KeeperErrorCode = NodeExists for /hadoop-ha/mynameservice/ActiveStandbyElectorLock",
      "2012-06-26 13:10:12,566 [myid:3] - DEBUG [ProcessThread(sid:3 cport:-1)::CommitProcessor@171] - Processing request:: sessionid:0x1382791d4e50004 type:createSession cxid:0x0 zxid:0x200000070 txntype:-10 reqpath:n/a",
      "2012-06-26 13:10:13,001 [myid:3] - DEBUG [ProcessThread(sid:3 cport:-1)::Leader@716] - Proposing:: sessionid:0x1382791d4e50004 type:createSession cxid:0x0 zxid:0x200000070 txntype:-10 reqpath:n/a",
      "2012-06-26 13:10:13,202 [myid:3] - DEBUG [LearnerHandler-/xx.xx.xx.102:13846:CommitProcessor@161] - Committing request:: sessionid:0x1382791d4e50004 type:createSession cxid:0x0 zxid:0x200000070 txntype:-10 reqpath:n/a",
      "2012-06-26 13:10:18,653 [myid:3] - INFO  [SessionTracker:ZooKeeperServer@325] - Expiring session 0x1382791d4e50004, timeout of 5000ms exceeded",
      "2012-06-26 13:10:18,803 [myid:3] - INFO  [ProcessThread(sid:3 cport:-1)::PrepRequestProcessor@627] - Got user-level KeeperException when processing sessionid:0x1382791d4e50004 type:create cxid:0x1 zxid:0x200000072 txntype:-1 reqpath:/hadoop-ha/hacluster/ActiveStandbyElectorLock Error Path:null Error:KeeperErrorCode = Session expired",
      "2012-06-26 13:10:18,834 [myid:3] - INFO  [ProcessThread(sid:3 cport:-1)::PrepRequestProcessor@476] - Processed session termination for sessionid: 0x1382791d4e50004",
      "2012-06-26 13:10:18,834 [myid:3] - DEBUG [ProcessThread(sid:3 cport:-1)::CommitProcessor@171] - Processing request:: sessionid:0x1382791d4e50004 type:closeSession cxid:0x0 zxid:0x200000074 txntype:-11 reqpath:n/a",
      "2012-06-26 13:10:19,892 [myid:3] - DEBUG [ProcessThread(sid:3 cport:-1)::Leader@716] - Proposing:: sessionid:0x1382791d4e50004 type:closeSession cxid:0x0 zxid:0x200000074 txntype:-11 reqpath:n/a",
      "2012-06-26 13:10:19,919 [myid:3] - DEBUG [LearnerHandler-/xx.xx.xx.102:13846:CommitProcessor@161] - Committing request:: sessionid:0x1382791d4e50004 type:closeSession cxid:0x0 zxid:0x200000074 txntype:-11 reqpath:n/a",
      "2012-06-26 13:10:20,608 [myid:3] - DEBUG [CommitProcessor:3:FinalRequestProcessor@88] - Processing request:: sessionid:0x1382791d4e50004 type:closeSession cxid:0x0 zxid:0x200000074 txntype:-11 reqpath:n/a",
      "2012-06-26 13:10:19,893 [myid:3] - DEBUG [ProcessThread(sid:3 cport:-1)::CommitProcessor@171] - Processing request:: sessionid:0x1382791d4e50004 type:create cxid:0x2 zxid:0x200000075 txntype:1 reqpath:n/a",
      "2012-06-26 13:10:19,920 [myid:3] - DEBUG [ProcessThread(sid:3 cport:-1)::Leader@716] - Proposing:: sessionid:0x1382791d4e50004 type:create cxid:0x2 zxid:0x200000075 txntype:1 reqpath:n/a",
      "2012-06-26 13:10:20,278 [myid:3] - DEBUG [LearnerHandler-/xx.xx.xx.102:13846:CommitProcessor@161] - Committing request:: sessionid:0x1382791d4e50004 type:create cxid:0x2 zxid:0x200000075 txntype:1 reqpath:n/a",
      "2012-06-26 13:10:20,608 [myid:3] - DEBUG [CommitProcessor:3:FinalRequestProcessor@88] - Processing request:: sessionid:0x1382791d4e50004 type:closeSession cxid:0x0 zxid:0x200000074 txntype:-11 reqpath:n/a",
      "2012-06-26 13:10:20,752 [myid:3] - DEBUG [CommitProcessor:3:FinalRequestProcessor@88] - Processing request:: sessionid:0x1382791d4e50004 type:create cxid:0x2 zxid:0x200000075 txntype:1 reqpath:n/a"
    ],
    "stack_traces": []
  },
  {
    "file": "ZooKeeper-1576.docx",
    "title": "ZooKeeper-1576",
    "version": "zookeeper-3.5.0",
    "Description": "Zookeeper cluster - failed to connect to cluster if one of the provided IPs causes java.net.UnknownHostException",
    "logs": [
      "Nov 02, 2012 9:54:32 PM com.netflix.curator.framework.imps.CuratorFrameworkImpl logError SEVERE: Background exception was not retry-able or retry gave up"
    ],
    "stack_traces": [
      "java.net.UnknownHostException: scnrmq003.myworkday.com\n\tat java.net.Inet4AddressImpl.lookupAllHostAddr(Native Method)\n\tat java.net.InetAddress$1.lookupAllHostAddr(Unknown Source)\n\tat java.net.InetAddress.getAddressesFromNameService(Unknown Source)\n\tat java.net.InetAddress.getAllByName0(Unknown Source)\n\tat java.net.InetAddress.getAllByName(Unknown Source)\n\tat java.net.InetAddress.getAllByName(Unknown Source)\n\tat org.apache.zookeeper.client.StaticHostProvider.<init>(StaticHostProvider.java:60)\n\tat org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:440)\n\tat org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:375)"
    ]
  },
  {
    "file": "ZooKeeper-1683.docx",
    "title": "ZooKeeper-1683",
    "version": "zookeeper-3.5.0",
    "Description": "ZooKeeper client NPE when updating server list on disconnected client",
    "logs": [
      "2013-04-04 22:16:15,872 ERROR [pool-4-thread-1] com.netflix.curator.ConnectionState.getZooKeeper (ConnectionState.java:84) - Background exception caught",
      "2013-04-10 05:19:52,787 WARN [main-SendThread(localhost:2181)] org.apache.zookeeper.ClientCnxn$SendThread.run (ClientCnxn.java:1122) - Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect",
      "2013-04-10 05:19:52,788 DEBUG [main-SendThread(localhost:2181)] org.apache.zookeeper.ClientCnxnSocketNIO.cleanup (ClientCnxnSocketNIO.java:193) - Ignoring exception during shutdown input",
      "2013-04-10 05:19:52,789 DEBUG [main-SendThread(localhost:2181)] org.apache.zookeeper.ClientCnxnSocketNIO.cleanup (ClientCnxnSocketNIO.java:200) - Ignoring exception during shutdown output"
    ],
    "stack_traces": [
      "java.lang.NullPointerException\n\tat org.apache.zookeeper.client.StaticHostProvider.updateServerList(StaticHostProvider.java:161) ~[zookeeper-3.5.0.jar:3.5.0--1]\n\tat org.apache.zookeeper.ZooKeeper.updateServerList(ZooKeeper.java:183) ~[zookeeper-3.5.0.jar:3.5.0--1]\n\tat com.netflix.curator.HandleHolder$1$1.setConnectionString(HandleHolder.java:121) ~[curator-client-1.3.5-SNAPSHOT.jar:?]",
      "java.net.ConnectException: Connection refused\n\tat sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.6.0_24]\n\tat sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:592) ~[?:1.6.0_24]\n\tat org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:353) ~[zookeeper-3.5.0.jar:3.5.0--1]\n\tat org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1101) [zookeeper-3.5.0.jar:3.5.0--1]",
      "java.nio.channels.ClosedChannelException\n\tat sun.nio.ch.SocketChannelImpl.shutdownInput(SocketChannelImpl.java:656) ~[?:1.6.0_24]\n\tat sun.nio.ch.SocketAdaptor.shutdownInput(SocketAdaptor.java:378) ~[?:1.6.0_24]\n\tat org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:190) [zookeeper-3.5.0.jar:3.5.0--1]\n\tat org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1190) [zookeeper-3.5.0.jar:3.5.0--1]\n\tat org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1130) [zookeeper-3.5.0.jar:3.5.0--1]",
      "java.nio.channels.ClosedChannelException\n\tat sun.nio.ch.SocketChannelImpl.shutdownOutput(SocketChannelImpl.java:667) ~[?:1.6.0_24]\n\tat sun.nio.ch.SocketAdaptor.shutdownOutput(SocketAdaptor.java:386) ~[?:1.6.0_24]\n\tat org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:197) [zookeeper-3.5.0.jar:3.5.0--1]\n\tat org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1190) [zookeeper-3.5.0.jar:3.5.0--1]\n\tat org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1130) [zookeeper-3.5.0.jar:3.5.0--1]"
    ]
  },
  {
    "file": "ZooKeeper-1732.docx",
    "title": "ZooKeeper-1732",
    "version": "zookeeper-3.4.5",
    "Description": "ZooKeeper server unable to join established ensemble.",
    "logs": [
      "2013-07-19 10:16:45,428 [myid:1] - WARN  [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:30101:NIOServerCnxn@354] - Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running",
      "2013-07-19 10:16:45,428 [myid:1] - DEBUG [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:30101:NIOServerCnxn@358] - IOException stack trace",
      "2013-07-19 10:16:45,428 [myid:1] - INFO  [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:30101:NIOServerCnxn@1001] - Closed socket connection for client /127.0.0.1:61907 (no session established for client)",
      "2013-07-19 10:17:00,833 [myid:1] - INFO [WorkerReceiver[myid=1]:FastLeaderElection@542] - Notification: 3 (n.leader), 0xb800000099 (n.zxid), 0xb9 (n.round), FOLLOWING (n.state), 2 (n.sid), 0xb8 (n.peerEPoch), LOOKING (my state)",
      "2013-07-19 10:17:00,833 [myid:1] - INFO [WorkerReceiver[myid=1]:FastLeaderElection@542] - Notification: 3 (n.leader), 0xb900000052 (n.zxid), 0xba (n.round), LEADING (n.state), 3 (n.sid), 0xb9 (n.peerEPoch), LOOKING (my state)"
    ],
    "stack_traces": [
      "java.io.IOException: ZooKeeperServer not running\n\tat org.apache.zookeeper.server.NIOServerCxn.readPayload(NIOServerCxn.java:352)\n\tat org.apache.zookeeper.server.NIOServerCxn.process(NIOServerCxn.java:313)\n\tat org.apache.zookeeper.server.NIOServerCxnFactory.run(NIOServerCxnFactory.java:199)\n\tat java.lang.Thread.run(Thread.java:619)"
    ]
  },
  {
    "file": "ZooKeeper-512.docx",
    "title": "ZooKeeper-512",
    "version": "zookeeper-3.2.0",
    "Description": "FLE election fails to elect leader. I was doing some fault injection testing of 3.2.1 with ZOOKEEPER-508 patch applied and noticed that after some time the ensemble failed to re-elect a leader.\n\nSee the attached log files - 5 member ensemble. typically 5 is the leader\n\nNotice that after 16:23:50,525 no quorum is formed, even after 20 minutes elapses w/no quorum\n\nenvironment:\n\nI was doing fault injection testing using aspectj. The faults are injected into socketchannel read/write, I throw exceptions randomly at a 1/200 ratio (rand.nextFloat() <= .005 => throw IOException",
    "logs": [
      "2009-08-19 16:05:25,447 - INFO  [SyncThread:1:ReadRequestFailsIntermittently@41] - READPACKET OK",
      "2009-08-19 16:05:25,447 - INFO  [SyncThread:1:ReadRequestFailsIntermittently@38] - READPACKET FORCED FAIL",
      "2009-08-19 16:05:25,448 - WARN  [SyncThread:1:SendAckRequestProcessor@46] - Closing connection to leader, exception during packet send",
      "2009-08-19 16:05:25,449 - INFO  [SyncThread:1:ReadRequestFailsIntermittently@41] - READPACKET OK",
      "2009-08-19 16:05:25,449 - INFO  [SyncThread:1:ReadRequestFailsIntermittently@41] - READPACKET OK",
      "2009-08-19 16:05:25,450 - WARN  [SyncThread:1:SendAckRequestProcessor@63] - Closing connection to leader, exception during packet send",
      "2009-08-19 16:05:25,456 - WARN  [QuorumPeer:/0:0:0:0:0:0:0:0:2181:Follower@309] - Exception when following the leader",
      "2009-08-19 16:05:25,457 - INFO  [QuorumPeer:/0:0:0:0:0:0:0:0:2181:NIOServerCnxn@833] - closing session:0x12334e7ba770000 NIOServerCnxn: java.nio.channels.SocketChannel[connected local=/127.0.0.1:2181 remote=/127.0.0.1:33152]",
      "2009-08-19 16:05:25,479 - INFO  [QuorumPeer:/0:0:0:0:0:0:0:0:2181:NIOServerCnxn@833] - closing session:0x12334e7226f0000 NIOServerCnxn: java.nio.channels.SocketChannel[connected local=/127.0.0.1:2181 remote=/127.0.0.1:33116]",
      "2009-08-19 16:05:25,546 - INFO  [Thread-3:ReadRequestFailsIntermittently@41] - READPACKET OK",
      "2009-08-19 16:05:25,546 - INFO  [Thread-3:ReadRequestFailsIntermittently@38] - READPACKET FORCED FAIL",
      "2009-08-19 16:05:25,546 - WARN  [Thread-3:QuorumCnxManager$RecvWorker@621] - Connection broken: ",
      "2009-08-19 16:05:51,057 - INFO  [Thread-8:ReadRequestFailsIntermittently@41] - READPACKET OK",
      "2009-08-19 16:05:51,059 - INFO  [QuorumPeer:/0:0:0:0:0:0:0:0:2181:FastLeaderElection@618] - Notification: 1, 4294967500, 2, 1, LOOKING, LOOKING, 1",
      "2009-08-19 16:05:51,059 - INFO  [QuorumPeer:/0:0:0:0:0:0:0:0:2181:FastLeaderElection@642] - Adding vote",
      "2009-08-19 16:05:51,060 - INFO  [Thread-2:ReadRequestFailsIntermittently@38] - READPACKET FORCED FAIL",
      "2009-08-19 16:05:51,060 - WARN  [Thread-2:QuorumCnxManager$SendWorker@550] - Exception when using channel: 2",
      "2009-08-19 16:05:51,061 - WARN  [Thread-2:QuorumCnxManager$SendWorker@554] - Send worker leaving thread",
      "2009-08-19 16:57:09,568 - INFO [Thread-74:ReadRequestFailsIntermittently@38] - READPACKET FORCED FAIL",
      "2009-08-25 10:51:04,617 - FATAL [SyncThread:2:SyncRequestProcessor@131] - Severe unrecoverable error, exiting"
    ],
    "stack_traces": [
      "java.io.IOException: aspect failed\n\tat org.apache.zookeeper.server.quorum.ReadRequestFailsIntermittently.ajc$before$org_apache_zookeeper_server_quorum_ReadRequestFailsIntermittently$1$61870ae2(ReadRequestFailsIntermittently.aj:39)\n\tat org.apache.zookeeper.server.quorum.Follower.writePacket(Follower.java:97)\n\tat org.apache.zookeeper.server.quorum.SendAckRequestProcessor.processRequest(SendAckRequestProcessor.java:44)\n\tat org.apache.zookeeper.server.SyncRequestProcessor.flush(SyncRequestProcessor.java:144)\n\tat org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:92)",
      "java.net.SocketException: Socket closed\n\tat java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:99)\n\tat java.net.SocketOutputStream.write(SocketOutputStream.java:136)\n\tat java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:65)\n\tat java.io.BufferedOutputStream.flush(BufferedOutputStream.java:123)\n\tat org.apache.zookeeper.server.quorum.Follower.writePacket(Follower.java:100)\n\tat org.apache.zookeeper.server.quorum.SendAckRequestProcessor.flush(SendAckRequestProcessor.java:61)\n\tat org.apache.zookeeper.server.SyncRequestProcessor.flush(SyncRequestProcessor.java:147)\n\tat org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:92)",
      "java.net.SocketException: Socket closed\n\tat java.net.SocketInputStream.read(SocketInputStream.java:162)\n\tat java.io.BufferedInputStream.fill(BufferedInputStream.java:218)\n\tat java.io.BufferedInputStream.read(BufferedInputStream.java:237)\n\tat java.io.DataInputStream.readInt(DataInputStream.java:370)\n\tat org.apache.jute.BinaryInputArchive.readInt(BinaryInputArchive.java:63)\n\tat org.apache.zookeeper.server.quorum.QuorumPacket.deserialize(QuorumPacket.java:66)\n\tat org.apache.jute.BinaryInputArchive.readRecord(BinaryInputArchive.java:108)\n\tat org.apache.zookeeper.server.quorum.Follower.readPacket(Follower.java:114)\n\tat org.apache.zookeeper.server.quorum.Follower.followLeader(Follower.java:243)\n\tat org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer.java:498)",
      "java.io.IOException: aspect failed\n\tat org.apache.zookeeper.server.quorum.ReadRequestFailsIntermittently.ajc$before$org_apache_zookeeper_server_quorum_ReadRequestFailsIntermittently$1$61870ae2(ReadRequestFailsIntermittently.aj:39)\n\tat org.apache.zookeeper.server.quorum.QuorumCnxManager$RecvWorker.run(QuorumCnxManager.java:594)",
      "java.io.IOException: aspect failed\n\tat org.apache.zookeeper.server.quorum.ReadRequestFailsIntermittently.ajc$before$org_apache_zookeeper_server_quorum_ReadRequestFailsIntermittently$1$61870ae2(ReadRequestFailsIntermittently.aj:39)\n\tat org.apache.zookeeper.server.quorum.QuorumCnxManager$SendWorker.send(QuorumCnxManager.java:512)\n\tat org.apache.zookeeper.server.quorum.QuorumCnxManager$SendWorker.run(QuorumCnxManager.java:548)",
      "java.net.SocketException: Socket closed\n\tat java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:99)\n\tat java.net.SocketOutputStream.write(SocketOutputStream.java:136)\n\tat java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:65)\n\tat java.io.BufferedOutputStream.flush(BufferedOutputStream.java:123)\n\tat org.apache.zookeeper.server.quorum.Follower.writePacket(Follower.java:100)\n\tat org.apache.zookeeper.server.quorum.SendAckRequestProcessor.flush(SendAckRequestProcessor.java:52)\n\tat org.apache.zookeeper.server.SyncRequestProcessor.flush(SyncRequestProcessor.java:147)\n\tat org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:92)"
    ]
  },
  {
    "file": "ZooKeeper-790.docx",
    "title": "ZooKeeper-790",
    "version": "zookeeper-3.3.1",
    "Description": "Last processed zxid set prematurely while establishing leadership",
    "logs": [
      "2010-06-20 23:22:46,421 - DEBUG [WorkerSender Thread:QuorumCnxManager@346] - Opening channel to server 1",
      "2010-06-20 23:22:46,423 - DEBUG [WorkerReceiver Thread:FastLeaderElection$Messenger$WorkerReceiver@214] - Receive new notification message. My id = 0",
      "2010-06-20 23:22:46,424 - INFO [QuorumPeer:/0.0.0.0:2181:FastLeaderElection@689] - Notification: 0, 77309411393, 1, 0, LOOKING, LOOKING, 0",
      "2010-06-20 23:22:46,424 - DEBUG [QuorumPeer:/0.0.0.0:2181:FastLeaderElection@495] - id: 0, proposed id: 0, zxid: 77309411393, proposed zxid: 77309411393",
      "2010-06-20 23:22:46,424 - DEBUG [QuorumPeer:/0.0.0.0:2181:FastLeaderElection@717] - Adding vote: From = 0, Proposed leader = 0, Porposed zxid = 77309411393, Proposed epoch = 1",
      "2010-06-20 23:22:46,426 - INFO [WorkerSender Thread:QuorumCnxManager@162] - Have smaller server identifier, so dropping the connection: (1, 0)",
      "2010-06-20 23:22:46,426 - DEBUG [WorkerSender Thread:QuorumCnxManager@346] - Opening channel to server 2",
      "2010-06-20 23:22:46,427 - DEBUG [Thread-1:QuorumCnxManager$Listener@445] - Connection request /192.168.1.182:46701",
      "2010-06-20 23:22:46,427 - DEBUG [Thread-1:QuorumCnxManager$Listener@448] - Connection request: 0",
      "2010-06-20 23:22:46,428 - DEBUG [Thread-1:QuorumCnxManager$SendWorker@504] - Address of remote peer: 1",
      "2010-06-20 23:22:46,428 - INFO [WorkerSender Thread:QuorumCnxManager@162] - Have smaller server identifier, so dropping the connection: (2, 0)",
      "2010-06-20 23:22:46,431 - DEBUG [WorkerReceiver Thread:FastLeaderElection$Messenger$WorkerReceiver@214] - Receive new notification message. My id = 0",
      "2010-06-20 23:22:46,432 - INFO [QuorumPeer:/0.0.0.0:2181:FastLeaderElection@689] - Notification: 1, 77309411372, 1, 0, LOOKING, LOOKING, 1",
      "2010-06-20 23:22:46,432 - DEBUG [QuorumPeer:/0.0.0.0:2181:FastLeaderElection@495] - id: 1, proposed id: 0, zxid: 77309411372, proposed zxid: 77309411393",
      "2010-06-20 23:22:46,432 - DEBUG [QuorumPeer:/0.0.0.0:2181:FastLeaderElection@717] - Adding vote: From = 1, Proposed leader = 1, Porposed zxid = 77309411372, Proposed epoch = 1",
      "2010-06-20 23:22:46,436 - DEBUG [Thread-1:QuorumCnxManager$Listener@445] - Connection request /192.168.1.183:44310",
      "2010-06-20 23:22:46,436 - DEBUG [Thread-1:QuorumCnxManager$Listener@448] - Connection request: 0",
      "2010-06-20 23:22:46,436 - DEBUG [Thread-1:QuorumCnxManager$SendWorker@504] - Address of remote peer: 2",
      "2010-06-20 23:22:46,440 - DEBUG [WorkerReceiver Thread:FastLeaderElection$Messenger$WorkerReceiver@214] - Receive new notification message. My id = 0",
      "2010-06-20 23:22:46,440 - INFO [QuorumPeer:/0.0.0.0:2181:FastLeaderElection@689] - Notification: 2, 73014444097, 1, 0, LOOKING, LOOKING, 2",
      "2010-06-20 23:22:46,440 - DEBUG [QuorumPeer:/0.0.0.0:2181:FastLeaderElection@495] - id: 2, proposed id: 0, zxid: 73014444097, proposed zxid: 77309411393",
      "2010-06-20 23:22:46,441 - DEBUG [QuorumPeer:/0.0.0.0:2181:FastLeaderElection@717] - Adding vote: From = 2, Proposed leader = 2, Porposed zxid = 73014444097, Proposed epoch = 1",
      "2010-06-20 23:22:46,441 - INFO [QuorumPeer:/0.0.0.0:2181:QuorumPeer@647] - LEADING",
      "2010-07-15 02:39:43,105 - INFO  [QuorumPeer:/0:0:0:0:0:0:0:0:2181:FileSnap@82] - Reading snapshot /data/zookeeper/version-2/snapshot.2300001ac2",
      "2010-07-15 02:39:43,321 - INFO  [QuorumPeer:/0:0:0:0:0:0:0:0:2181:FastLeaderElection@649] - New election. My id =  1, Proposed zxid = 154618826848       ",
      "2010-07-15 02:39:43,322 - INFO  [QuorumPeer:/0:0:0:0:0:0:0:0:2181:FastLeaderElection@689] - Notification: 1, 154618826848, 4, 1, LOOKING, LOOKING, 1          ",
      "2010-07-15 02:39:43,325 - INFO  [QuorumPeer:/0:0:0:0:0:0:0:0:2181:FastLeaderElection@799] - Notification: 2, 146030952153, 3, 1, LOOKING, LEADING, 2",
      "2010-07-15 02:39:43,326 - INFO  [QuorumPeer:/0:0:0:0:0:0:0:0:2181:FastLeaderElection@799] - Notification: 2, 146030952153, 3, 1, LOOKING, FOLLOWING, 3",
      "2010-07-15 02:39:43,326 - INFO  [QuorumPeer:/0:0:0:0:0:0:0:0:2181:QuorumPeer@642] - FOLLOWING",
      "2010-07-15 02:39:43,339 - FATAL [QuorumPeer:/0:0:0:0:0:0:0:0:2181:Follower@71] - Leader epoch 23 is less than our epoch 24",
      "2010-07-15 02:39:43,339 - WARN  [QuorumPeer:/0:0:0:0:0:0:0:0:2181:Follower@82] - Exception when following the leader",
      "2010-07-15 02:39:43,340 - INFO  [QuorumPeer:/0:0:0:0:0:0:0:0:2181:Follower@166] - shutdown called",
      "2010-07-15 02:39:43,340 - INFO  [QuorumPeer:/0:0:0:0:0:0:0:0:2181:QuorumPeer@620] - LOOKING",
      "2010-07-15 02:35:36,398 - INFO  [QuorumPeer:/0:0:0:0:0:0:0:0:2181:NIOServerCnxn@1661] - Established session 0x229aa13cfc6276b with negotiated timeout 10000 for client /10.209.45.114:34562",
      "2010-07-15 02:39:18,907 - INFO  [main:QuorumPeerConfig@90] - Reading configuration from: /etc/zookeeper/conf/zoo.cfg",
      "2010-07-15 02:39:43,339 - FATAL [QuorumPeer:/0:0:0:0:0:0:0:0:2181:Follower@71] - Leader epoch 23 is less than our epoch 24",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:QuorumPeer@654] - LEADING",
      "INFO  - [WorkerReceiver Thread:FastLeaderElection@496] - Notification: 3 (n.leader), 0 (n.zxid), 1 (n.round), LEADING (n.state), 3 (n.sid), LOOKING (my state)",
      "INFO  - [WorkerReceiver Thread:FastLeaderElection@496] - Notification: 3 (n.leader), 0 (n.zxid), 1 (n.round), LEADING (n.state), 3 (n.sid), LEADING (my state)",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Leader@54] - TCP NoDelay set to: true",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:zookeeper.version=3.4.0--1, built on 07/15/2010 10:36 GMT",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:host.name=XXXXXXXXXX.com",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:java.version=1.6.0_04",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:java.vendor=Sun Microsystems Inc.",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:java.home=/usr/java/jdk1.6.0_04/jre",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:java.class.path=.XXXXXXXXX",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:java.library.path= XXXXXXXXX",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:java.io.tmpdir=/tmp",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:java.compiler=<NA>",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:os.name=Linux",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:os.arch=amd64",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:os.version=2.6.18-53.1.21.el5",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:user.name=XXXXXXXXX",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:user.home=XXXXXXXXX",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:user.dir=XXXXXXXXX",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:ZooKeeperServer@151] - Created server with tickTime 3000 minSessionTimeout 6000 maxSessionTimeout 60000 datadir /XXXXXXXXX/zookeeper/version-2 snapdir /XXXXXXXX/zookeeper/version-2",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:FileSnap@82] - Reading snapshot /XXXXXXXX/zookeeper/version-2/snapshot.100113340",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:FileSnap@82] - Reading snapshot /XXXXXXXX/zookeeper/version-2/snapshot.100113340",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:FileTxnSnapLog@208] - Snapshotting: 10011f748",
      "INFO  - [SessionTracker:ZooKeeperServer@315] - Expiring session 0x229d6a9e0ca0000, timeout of 10000ms exceeded",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Leader@394] - Shutdown called",
      "INFO  - [Thread-10:Leader$LearnerCnxAcceptor@243] - exception while shutting down acceptor: java.net.SocketException: Socket closed",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:FinalRequestProcessor@378] - shutdown of request processor complete",
      "INFO  - [SyncThread:2:SyncRequestProcessor@151] - SyncRequestProcessor exited!",
      "INFO  - [CommitProcessor:2:CommitProcessor@148] - CommitProcessor exited loop!",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:QuorumPeer@620] - LOOKING",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:FileSnap@82] - Reading snapshot /XXXXXXXXX/zookeeper/version-2/snapshot.10011f748",
      "INFO  - [SessionTracker:SessionTrackerImpl@158] - SessionTrackerImpl exited loop!",
      "INFO  - [ProcessThread:-1:PrepRequestProcessor@385] - Processed session termination for sessionid: 0x229d6a9e0ca0000",
      "ERROR - [ProcessThread:-1:NIOServerCnxn$Factory$1@87] - Thread Thread[ProcessThread:-1,5,main] died",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:FastLeaderElection@663] - New election. My id =  2, Proposed zxid = 4296144712",
      "INFO  - [WorkerReceiver Thread:FastLeaderElection@496] - Notification: 2 (n.leader), 4296144712 (n.zxid), 2 (n.round), LOOKING (n.state), 2 (n.sid), LOOKING (my state)",
      "INFO  - [WorkerSender Thread:QuorumCnxManager@162] - Have smaller server identifier, so dropping the connection: (3, 2)",
      "INFO  - [WorkerReceiver Thread:FastLeaderElection@496] - Notification: 3 (n.leader), 0 (n.zxid), 1 (n.round), LEADING (n.state), 3 (n.sid), LOOKING (my state)",
      "INFO  - [WorkerReceiver Thread:FastLeaderElection@496] - Notification: 3 (n.leader), 0 (n.zxid), 1 (n.round), FOLLOWING (n.state), 1 (n.sid), LOOKING (my state)",
      "INFO  - [WorkerReceiver Thread:FastLeaderElection@496] - Notification: 3 (n.leader), 0 (n.zxid), 1 (n.round), FOLLOWING (n.state), 1 (n.sid), LOOKING (my state)",
      "INFO  - [WorkerReceiver Thread:FastLeaderElection@496] - Notification: 3 (n.leader), 0 (n.zxid), 1 (n.round), FOLLOWING (n.state), 1 (n.sid), LOOKING (my state)",
      "INFO  - [WorkerReceiver Thread:FastLeaderElection@496] - Notification: 3 (n.leader), 0 (n.zxid), 1 (n.round), LEADING (n.state), 3 (n.sid), LOOKING (my state)",
      "INFO  - [WorkerReceiver Thread:FastLeaderElection@496] - Notification: 3 (n.leader), 0 (n.zxid), 1 (n.round), LEADING (n.state), 3 (n.sid), LOOKING (my state)",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:QuorumPeer@642] - FOLLOWING",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Learner@72] - TCP NoDelay set to: true",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:ZooKeeperServer@151] - Created server with tickTime 3000 minSessionTimeout 6000 maxSessionTimeout 60000 datadir /XXXXXXXXX/zookeeper/version-2 snapdir /XXXXXXXXX/zookeeper/version-2",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Learner@285] - Getting a snapshot from leader",
      "FATAL - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Learner@314] - Setting leader epoch 1",
      "WARN  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Follower@116] - Got zxid 0x100162711 expected 0x1",
      "WARN  - [SyncThread:2:FileTxnLog@196] - Creating new log file: 100162711",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:FileTxnSnapLog@208] - Snapshotting: 100164c43",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:QuorumPeer@654] - LEADING",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Leader@54] - TCP NoDelay set to: true",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:zookeeper.version=3.4.0--1, built on 07/15/2010 10:36 GMT",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:host.name=XXXXXXXX.com",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:java.version=1.6.0_05",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:java.vendor=Sun Microsystems Inc.",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:java.home=/usr/java/jdk1.6.0_05/jre",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:java.class.path=XXXXXXXXX",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:java.library.path=XXXXXXXXX",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:java.io.tmpdir=/tmp",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:java.compiler=<NA>",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:os.name=Linux",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:os.arch= amd64",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:os.version=2.6.18-92.1.13.el5.perfctr.2.6.36",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:user.name=XXXXXXXXX",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:user.home=XXXXXXXX",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Environment@97] - Server environment:user.dir=XXXXXXXXX",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:ZooKeeperServer@151] - Created server with tickTime 3000 minSessionTimeout 6000 maxSessionTimeout 60000 datadir /xxxxxxx/zookeeper/version-2 snapdir /XXXXXXX/zookeeper/version-2",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:FileSnap@82] - Reading snapshot /XXXXXXX/zookeeper/version-2/snapshot.1000469b4",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:FileSnap@82] - Reading snapshot /XXXXXXX/zookeeper/version-2/snapshot.1000469b4",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:FileTxnSnapLog@208] - Snapshotting: 100055cec",
      "INFO  - [SessionTracker:ZooKeeperServer@315] - Expiring session 0x129d6b61b5b0000, timeout of 10000ms exceeded",
      "INFO  - [ProcessThread:-1:PrepRequestProcessor@385] - Processed session termination for sessionid: 0x129d6b61b5b0000",
      "WARN  - [SyncThread:1:FileTxnLog@196] - Creating new log file: log.200000001",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Leader@394] - Shutdown called",
      "INFO  - [ProcessThread:-1:PrepRequestProcessor@119] - PrepRequestProcessor exited loop!",
      "INFO  - [CommitProcessor:1:CommitProcessor@148] - CommitProcessor exited loop!",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:FinalRequestProcessor@378] - shutdown of request processor complete",
      "INFO  - [SyncThread:1:SyncRequestProcessor@151] - SyncRequestProcessor exited!",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:QuorumPeer@620] - LOOKING",
      "INFO  - [Thread-8:Leader$LearnerCnxAcceptor@243] - exception while shutting down acceptor: java.net.SocketException: Socket closed",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:FileSnap@82] - Reading snapshot /XXXXXXXX/zookeeper/version-2/snapshot.100055cec",
      "INFO  - [SessionTracker:SessionTrackerImpl@158] - SessionTrackerImpl exited loop!",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:FastLeaderElection@663] - New election. My id =  1, Proposed zxid = 8589934593",
      "INFO  - [WorkerReceiver Thread:FastLeaderElection@496] - Notification: 1 (n.leader), 8589934593 (n.zxid), 2 (n.round), LOOKING (n.state), 1 (n.sid), LOOKING (my state)",
      "INFO  - [WorkerSender Thread:QuorumCnxManager@162] - Have smaller server identifier, so dropping the connection: (2, 1)",
      "INFO  - [WorkerReceiver Thread:FastLeaderElection@496] - Notification: 3 (n.leader), 0 (n.zxid), 1 (n.round), FOLLOWING (n.state), 2 (n.sid), LOOKING (my state)",
      "INFO  - [WorkerReceiver Thread:FastLeaderElection@496] - Notification: 3 (n.leader), 0 (n.zxid), 1 (n.round), LEADING (n.state), 3 (n.sid), LOOKING (my state)",
      "INFO  - [WorkerReceiver Thread:FastLeaderElection@496] - Notification: 3 (n.leader), 0 (n.zxid), 1 (n.round), FOLLOWING (n.state), 2 (n.sid), LOOKING (my state)",
      "INFO  - [WorkerReceiver Thread:FastLeaderElection@496] - Notification: 3 (n.leader), 0 (n.zxid), 1 (n.round), FOLLOWING (n.state), 2 (n.sid), LOOKING (my state)",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:QuorumPeer@642] - FOLLOWING",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Learner@72] - TCP NoDelay set to: true",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:ZooKeeperServer@151] - Created server with tickTime 3000 minSessionTimeout 6000 maxSessionTimeout 60000 datadir /xxxxxxx/zookeeper/version-2 snapdir /xxxxxxx/zookeeper/version-2",
      "FATAL - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Follower@71] - Leader epoch 1 is less than our epoch 2",
      "WARN  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Follower@82] - Exception when following the leader",
      "INFO  - [QuorumPeer:/0:0:0:0:0:0:0:0:10218:Follower@166] - shutdown called"
    ],
    "stack_traces": [
      "java.io.IOException: Error: Epoch of leader is lower\n\tat org.apache.zookeeper.server.quorum.Follower.followLeader(Follower.java:73)\n\tat org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer.java:644)",
      "java.lang.Exception: shutdown Follower\n\tat org.apache.zookeeper.server.quorum.Follower.shutdown(Follower.java:166)\n\tat org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer.java:648)",
      "java.lang.Exception: shutdown Leader! reason: Waiting for a quorum of followers, only synced with: 2: \n\tat org.apache.zookeeper.server.quorum.Leader.shutdown(Leader.java:394)\n\tat org.apache.zookeeper.server.quorum.Leader.lead(Leader.java:317)\n\tat org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer.java:657)",
      "java.net.SocketException: Socket closed\n\tat java.net.ServerSocket.close(ServerSocket.java:466)\n\tat org.apache.zookeeper.server.quorum.Leader$LearnerCnxAcceptor.shutdown(Leader.java:241)\n\tat org.apache.zookeeper.server.quorum.Leader.shutdown(Leader.java:398)\n\tat org.apache.zookeeper.server.quorum.Leader.lead(Leader.java:317)\n\tat org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer.java:657)",
      "java.lang.NullPointerException\n\tat org.apache.zookeeper.server.quorum.ProposalRequestProcessor.processRequest(ProposalRequestProcessor.java:71)\n\tat org.apache.zookeeper.server.PrepRequestProcessor.pRequest(PrepRequestProcessor.java:435)\n\tat org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:114)",
      "java.lang.Exception: shutdown Leader! reason: Waiting for a quorum of followers, only synced with: 1: \n\tat org.apache.zookeeper.server.quorum.Leader.shutdown(Leader.java:394)\n\tat org.apache.zookeeper.server.quorum.Leader.lead(Leader.java:317)\n\tat org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer.java:657)",
      "java.net.SocketException: Socket closed\n\tat java.net.ServerSocket.close(ServerSocket.java:466)\n\tat org.apache.zookeeper.server.quorum.Leader$LearnerCnxAcceptor.shutdown(Leader.java:241)\n\tat org.apache.zookeeper.server.quorum.Leader.shutdown(Leader.java:398)\n\tat org.apache.zookeeper.server.quorum.Leader.lead(Leader.java:317)\n\tat org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer.java:657)",
      "java.io.IOException: Error: Epoch of leader is lower\n\tat org.apache.zookeeper.server.quorum.Follower.followLeader(Follower.java:73)\n\tat org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer.java:644)",
      "java.lang.Exception: shutdown Follower\n\tat org.apache.zookeeper.server.quorum.Follower.shutdown(Follower.java:166)\n\tat org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer.java:648)"
    ]
  }
]