00:00:50  * groglogicquit (Ping timeout: 245 seconds)
00:10:41  * fallsemojoined
00:12:20  * timoxleyjoined
00:17:13  * timoxleyquit (Ping timeout: 272 seconds)
00:21:38  * scttnlsnjoined
00:32:09  * thlorenz_joined
00:34:11  * tmcwquit (Remote host closed the connection)
00:34:45  * tmcwjoined
00:34:56  * jjmalinaquit (Quit: Leaving.)
00:36:31  * thlorenz_quit (Ping timeout: 246 seconds)
00:47:58  * paulfryzelquit (Remote host closed the connection)
00:51:04  * tmcwquit (Remote host closed the connection)
00:58:17  * esundahlquit (Remote host closed the connection)
00:58:45  * esundahljoined
01:01:14  * jxsonquit (Remote host closed the connection)
01:01:43  * jxsonjoined
01:02:04  * jxsonquit (Read error: Connection reset by peer)
01:02:25  * jxsonjoined
01:02:48  * esundahlquit (Ping timeout: 240 seconds)
01:04:08  * groglogi_quit (Remote host closed the connection)
01:07:37  * jxsonquit (Ping timeout: 272 seconds)
01:13:01  * timoxleyjoined
01:14:04  * ednapira_joined
01:16:54  * timoxleyquit (Read error: Connection reset by peer)
01:17:25  * timoxleyjoined
01:18:58  * paulfryzeljoined
01:23:52  * paulfryzelquit (Ping timeout: 264 seconds)
01:29:04  * fallsemoquit (Ping timeout: 265 seconds)
01:32:59  * thlorenzjoined
01:33:37  * fallsemojoined
01:35:39  * dguttmanquit (Quit: dguttman)
01:37:39  * thlorenzquit (Ping timeout: 272 seconds)
01:38:39  * fallsemoquit (Ping timeout: 272 seconds)
01:40:05  * fallsemojoined
01:49:08  * fallsemoquit (Ping timeout: 240 seconds)
01:55:59  * scttnlsnquit (Quit: Leaving...)
02:00:35  * fallsemojoined
02:01:41  * tmcwjoined
02:05:37  * fallsemoquit (Ping timeout: 268 seconds)
02:06:24  * tmcwquit (Ping timeout: 260 seconds)
02:13:28  * groglogicjoined
02:13:29  * soldairquit (Ping timeout: 250 seconds)
02:22:45  * sneakynessquit (Quit: ¡)
02:33:52  * thlorenzjoined
02:38:52  * thlorenzquit (Ping timeout: 264 seconds)
02:43:54  * fallsemojoined
02:46:10  * timoxleyquit
02:48:07  * fallsemoquit (Ping timeout: 240 seconds)
02:50:36  * tarrudaquit (Remote host closed the connection)
02:50:45  * ednapira_quit (Remote host closed the connection)
02:52:44  * jcrugzzjoined
03:04:40  * esundahljoined
03:17:35  * jjmalinajoined
03:17:44  * jjmalinaquit (Client Quit)
03:17:57  * ednapiranhajoined
03:27:08  * wolfeidauquit (Ping timeout: 261 seconds)
03:33:11  * wolfeidaujoined
03:34:37  * thlorenzjoined
03:36:30  * thlorenz_joined
03:36:30  * thlorenzquit (Read error: Connection reset by peer)
03:38:08  * wolfeidauquit (Ping timeout: 240 seconds)
03:41:09  * thlorenz_quit (Ping timeout: 272 seconds)
03:55:10  * mikealjoined
03:58:12  * wolfeidaujoined
04:02:09  * ednapiranhaquit (Quit: Leaving...)
04:16:09  * mesch_joined
04:19:32  * chilts_joined
04:22:03  * mmckegg_joined
04:22:24  * trevnorris_joined
04:23:30  * chiltsquit (*.net *.split)
04:23:30  * meschquit (*.net *.split)
04:23:31  * mmckeggquit (*.net *.split)
04:23:33  * trevnorrisquit (*.net *.split)
04:23:33  * trevnorris_changed nick to trevnorris
04:23:34  * mmckegg_changed nick to mmckegg
04:23:38  * mesch_changed nick to mesch
04:28:26  * jxsonjoined
04:31:30  * mikealquit (Quit: Leaving.)
04:31:40  * mikealjoined
04:34:20  * mikealquit (Client Quit)
04:37:17  * thlorenzjoined
04:38:41  * fallsemojoined
04:39:11  * esundahl_joined
04:40:12  * esundahlquit (Read error: Connection reset by peer)
04:42:01  * fallsemoquit (Read error: Operation timed out)
04:42:19  * thlorenzquit (Ping timeout: 272 seconds)
05:00:38  * dguttmanjoined
05:04:46  * fallsemojoined
05:10:15  * jxsonquit (Remote host closed the connection)
05:12:50  * fallsemoquit (Quit: Leaving.)
05:22:26  * dguttmanquit (Quit: dguttman)
05:25:56  * tmcwjoined
05:38:06  * thlorenzjoined
05:42:30  * thlorenzquit (Ping timeout: 245 seconds)
05:50:08  * mikealjoined
05:50:46  * tmcwquit
06:06:04  * jxsonjoined
06:11:53  * wolfeidauquit (Ping timeout: 272 seconds)
06:17:01  * groglogicquit (Remote host closed the connection)
06:18:01  * jxsonquit (Remote host closed the connection)
06:18:37  * jxsonjoined
06:22:07  * wolfeidaujoined
06:23:39  * jxsonquit (Ping timeout: 272 seconds)
06:38:59  * thlorenzjoined
06:43:13  * thlorenzquit (Ping timeout: 240 seconds)
06:47:56  * jxsonjoined
06:49:18  * jxsonquit (Remote host closed the connection)
06:52:51  * jxson_joined
07:26:36  * jxson_quit (Remote host closed the connection)
07:27:46  * mcollinaquit (Read error: Connection reset by peer)
07:27:46  * fb55quit (Read error: Connection reset by peer)
07:27:52  * mcollina_joined
07:28:21  * fb55joined
07:31:34  * jxsonjoined
07:32:08  * esundahl_quit (Remote host closed the connection)
07:39:47  * thlorenzjoined
07:44:16  * thlorenzquit (Ping timeout: 264 seconds)
08:04:35  * fb55quit (Read error: Connection reset by peer)
08:38:16  * fb55joined
08:40:37  * thlorenz_joined
08:45:28  * thlorenz_quit (Ping timeout: 264 seconds)
08:49:06  * jxsonquit (Remote host closed the connection)
08:52:32  * frankblizzardjoined
08:56:10  * fb55quit (Remote host closed the connection)
08:59:22  * timoxleyjoined
09:41:27  * thlorenzjoined
09:46:04  * thlorenzquit (Ping timeout: 264 seconds)
10:16:45  * tarrudajoined
10:41:25  * jcrugzzquit (Ping timeout: 272 seconds)
10:42:19  * thlorenzjoined
10:47:07  * thlorenzquit (Ping timeout: 272 seconds)
11:07:25  * jcrugzzjoined
11:15:50  * jcrugzzquit (Ping timeout: 245 seconds)
11:43:07  * thlorenzjoined
11:47:28  * thlorenzquit (Ping timeout: 246 seconds)
12:43:54  * thlorenzjoined
12:48:28  * thlorenzquit (Ping timeout: 264 seconds)
12:50:52  * fallsemojoined
13:03:23  * mcollina_quit (Remote host closed the connection)
13:03:59  * mcollinajoined
13:08:59  * mcollinaquit (Ping timeout: 272 seconds)
13:17:11  * thlorenzjoined
13:22:30  * fallsemoquit (Ping timeout: 245 seconds)
13:34:30  * mcollinajoined
13:38:54  * mcollinaquit (Ping timeout: 246 seconds)
13:50:12  * tarrudaquit (Remote host closed the connection)
13:57:54  * tmcwjoined
14:04:30  * mcollinajoined
14:06:08  * mcollinaquit (Read error: Connection reset by peer)
14:06:13  * mcollina_joined
14:10:24  * mcollina_quit (Ping timeout: 246 seconds)
14:12:35  * tmcwquit (Remote host closed the connection)
14:13:07  * tmcwjoined
14:17:39  * tmcwquit (Ping timeout: 256 seconds)
14:19:52  * thlorenzquit (Remote host closed the connection)
14:21:02  * fallsemojoined
14:38:16  * tmcwjoined
14:39:46  * mcollinajoined
14:48:20  <ogd>rescrv: code is kind of messy but I implemented db copying using .liveBackup https://github.com/maxogden/dat/blob/master/lib/commands.js#L160
14:48:35  <ogd>rescrv: that was the server code, heres the client https://github.com/maxogden/dat/blob/master/lib/commands.js#L88
14:49:01  <ogd>rescrv: basically i create a backup, then stream it as a .tar.gz to the client, who untars it. afterwards I delete the backup folder. works pretty well!
14:49:34  <ogd>rescrv: have a simple benchmark here https://github.com/maxogden/dat/blob/master/test/bulk-replication.js#L1-L15
14:50:17  <rescrv>ogd: good to hear!
14:51:11  <ogd>rescrv: I did it with a million key db and worked a charm as well, .liveBackup is pretty quick
14:52:13  <rescrv>ogd: it basically uses the mechanism employed by "snapshot", then operates on that
14:52:51  <rescrv>you can also rsync the livebackup and take advantage of its builtin diff algorithm
14:54:43  * dguttmanjoined
14:54:46  <rescrv>you can do this yourself for db copying by omitting any *.sst files that exist on both sids
14:55:14  <ogd>rescrv: ah, .ssts never change once they are written?
14:55:37  <rescrv>ogd: correct
14:56:16  <rescrv>that's why you can backup a 1TB database in literally seconds
14:56:24  <ogd>this is a nice option... basically if its okay that you bring your database down you can get super fast sync
14:56:36  <ogd>whereas if you want to keep it available i can offer a networked sync that is slower
14:57:49  <ogd>(networked meaning serailized data through the leveldb api)
14:59:12  <rescrv>ogd: the cost of LiveBckup is proportional to a write, plus copying 100KB + write_buffer_size
14:59:38  <rescrv>By "cost" I mean the time that your DB is unavailable for writes
15:00:04  <rescrv>the remaining time in that "seconds" number is not going to hold up writes
15:00:38  <ogd>rescrv: i was thinking in terms of the receiving end in a replication scenario, it could get tricky to merge in remote updates consistently
15:00:44  * thlorenzjoined
15:01:15  <rescrv>ogd: I think if you are doing replication, you can copy the backup and then do a merge with two iterators
15:01:47  <rescrv>there's other features in HyperLevelDB that make replication cheap, but they require knowing the code well enough that you'd be better looking at the code than asking me
15:02:10  <rescrv>we use them in HyperDex to take a node offline and copy only the data that it missed in the time it was offline
15:02:24  <ogd>rescrv: ahh very good point. also I could probably do a reverse read on the replica since theyre presorted
15:04:05  <rescrv>ogd: that too
15:04:18  <rescrv>is your goal occasional replication or online replication?
15:04:57  <ogd>rescrv: occasional with potentially large deltas. node is well suited for online replication
15:06:29  <ogd>rescrv: also somethign I anticipate my users wanting is a way of doing distributed map reduce, so taking one large leveldb and spinning up N nodes, sharding it N ways, doing some distributed job, then replicating results back, shutting down the N nodes, etc
15:11:06  <rescrv>ogd: for the map reduce, it's often not efficient to fan the data over the network from one source. If the data is small,you can just compute locally. If the data is large, you spend more time shipping it than computing on it. If you're doing that you probably want to always keep it sharded
15:11:46  * jjmalinajoined
15:11:47  <ogd>rescrv: ahh good point!
15:15:39  * brianloveswordsquit (Excess Flood)
15:17:12  * brianloveswordsjoined
15:21:02  * thlorenz_joined
15:24:26  * ryan_ramagejoined
15:25:18  * thlorenz_quit (Ping timeout: 246 seconds)
15:33:21  * brianloveswordsquit (Excess Flood)
15:35:11  * brianloveswordsjoined
15:48:45  * ednapiranhajoined
15:55:11  * jerrysvjoined
15:55:50  * mikealquit (Quit: Leaving.)
16:01:43  * sneakynessjoined
16:05:43  * paulfryzeljoined
16:09:51  * tmcwquit (Remote host closed the connection)
16:11:13  * tmcwjoined
16:20:13  * esundahljoined
16:21:01  * thlorenz_joined
16:22:13  * mikealjoined
16:24:53  * mikealquit (Client Quit)
16:25:22  * thlorenz_quit (Ping timeout: 246 seconds)
16:26:45  * mikealjoined
16:30:17  * mikealquit (Client Quit)
16:35:25  * mikealjoined
16:38:11  * mikealquit (Client Quit)
16:40:28  * mikealjoined
16:51:40  * mikealquit (Quit: Leaving.)
16:59:13  * mikealjoined
17:04:35  * tmcwquit (Remote host closed the connection)
17:05:37  * tmcwjoined
17:16:25  * frankblizzardquit
17:21:50  * thlorenz_joined
17:23:52  * fb55joined
17:26:10  * groglogicjoined
17:26:14  * rudjoined
17:26:23  * mikealquit (Quit: Leaving.)
17:26:29  * thlorenz_quit (Ping timeout: 248 seconds)
17:26:53  * fb55quit (Remote host closed the connection)
17:34:52  * rudquit (Ping timeout: 244 seconds)
17:42:10  * rudjoined
17:42:10  * rudquit (Changing host)
17:42:10  * rudjoined
17:51:13  * rudquit (Ping timeout: 240 seconds)
17:54:37  * mcollinaquit (Ping timeout: 272 seconds)
17:55:12  * mcollinajoined
17:56:11  * rudjoined
18:04:44  * sneakynessquit (Quit: ¡)
18:05:55  * rudquit (Ping timeout: 260 seconds)
18:10:17  * rudjoined
18:13:24  * soldairjoined
18:14:44  * mikealjoined
18:14:54  * rud_joined
18:14:54  * rud_quit (Changing host)
18:14:54  * rud_joined
18:15:50  * rudquit (Ping timeout: 265 seconds)
18:16:18  * rudjoined
18:16:19  * rudquit (Changing host)
18:16:19  * rudjoined
18:16:20  * rudquit (Client Quit)
18:19:27  * mikealquit (Ping timeout: 260 seconds)
18:19:40  * rud_quit (Ping timeout: 264 seconds)
18:22:25  * astrolinquit (Remote host closed the connection)
18:22:28  * ELLIOTTCABLEquit (Remote host closed the connection)
18:22:40  * thlorenz_joined
18:23:18  * mikealjoined
18:25:05  * Raynosquit (Remote host closed the connection)
18:25:06  * sveisveiquit (Remote host closed the connection)
18:25:06  * ehdquit (Remote host closed the connection)
18:25:06  * thefoxisquit (Remote host closed the connection)
18:25:06  * justicefriesquit (Remote host closed the connection)
18:27:49  * thlorenz_quit (Ping timeout: 272 seconds)
18:28:29  * meschquit (Remote host closed the connection)
18:35:04  * ehdjoined
18:35:50  * jxsonjoined
18:42:47  * justicefriesjoined
18:45:52  * ELLIOTTCABLEjoined
18:46:58  * sveisveijoined
18:47:15  * Raynosjoined
18:49:09  * mikealquit (Ping timeout: 248 seconds)
18:50:43  * groglogicquit (Ping timeout: 260 seconds)
18:55:00  * groglogicjoined
18:57:22  * thefoxisjoined
18:58:55  * mikealjoined
19:05:00  * alessioalexjoined
19:07:08  * meschjoined
19:10:40  * astrolinjoined
19:11:06  * alessioalexchanged nick to alessioalex__
19:11:38  * alessioalex__quit (Quit: Page closed)
19:12:56  * thefoxisquit (Read error: Connection reset by peer)
19:15:16  * thefoxisjoined
19:21:24  * meschquit (Read error: Connection reset by peer)
19:22:12  * meschjoined
19:23:22  * thlorenz_joined
19:28:15  * thlorenz_quit (Ping timeout: 272 seconds)
19:46:08  * scttnlsnjoined
19:47:59  * mikeal1joined
19:48:39  * mikeal1quit (Read error: Connection reset by peer)
19:50:09  * mikeal1joined
19:50:44  * mikealquit (Ping timeout: 244 seconds)
19:54:03  * groglogicquit (Remote host closed the connection)
19:57:41  * nialloZ_joined
20:00:40  * Punnajoined
20:03:11  * mikeal1quit (Quit: Leaving.)
20:05:21  * chrisdickinson_joined
20:09:29  * ryan_ramagequit (Quit: ryan_ramage)
20:09:32  * nialloZquit (Ping timeout: 240 seconds)
20:09:32  * Pwnnaquit (Ping timeout: 240 seconds)
20:09:33  * chrisdickinsonquit (Quit: ZNC - http://znc.in)
20:09:35  * Punnachanged nick to Pwnna
20:10:32  * chrisdickinson_changed nick to chrisdickinson
20:12:01  * jxsonquit (Remote host closed the connection)
20:12:27  * jxsonjoined
20:17:09  * jxsonquit (Ping timeout: 265 seconds)
20:21:04  * scttnlsnquit (Quit: Leaving...)
20:24:11  * thlorenz_joined
20:25:00  * groglogicjoined
20:28:38  * thlorenz_quit (Ping timeout: 240 seconds)
20:31:58  * mcollinaquit (Remote host closed the connection)
20:32:35  * mcollinajoined
20:37:07  * mcollinaquit (Ping timeout: 260 seconds)
21:02:26  * tarrudajoined
21:03:08  * mcollinajoined
21:07:58  * mcollinaquit (Ping timeout: 268 seconds)
21:08:31  * jxsonjoined
21:10:24  * jcrugzzjoined
21:25:02  * thlorenz_joined
21:29:33  * thlorenz_quit (Ping timeout: 268 seconds)
21:34:38  * mcollinajoined
21:39:19  * mcollinaquit (Ping timeout: 265 seconds)
21:39:48  * mcollinajoined
21:52:15  * mikealjoined
21:53:55  * vincentmacjoined
22:25:56  * thlorenz_joined
22:30:14  * thlorenz_quit (Ping timeout: 240 seconds)
22:33:37  * thlorenzquit (Remote host closed the connection)
23:03:28  * groglogicquit (Ping timeout: 244 seconds)
23:04:32  * thlorenzjoined
23:05:58  * groglogicjoined
23:12:38  * thlorenzquit (Ping timeout: 240 seconds)
23:19:55  * zz_eugenewarechanged nick to eugeneware
23:21:39  * thlorenzjoined
23:26:43  * thlorenzquit (Ping timeout: 268 seconds)
23:38:31  * tarrudaquit (Quit: WeeChat 0.4.2)
23:45:06  * jjmalinaquit (Quit: Leaving.)
23:55:05  * tarrudajoined
23:55:55  * tmcwquit (Remote host closed the connection)
23:56:29  * tmcwjoined
23:57:47  * fallsemoquit (Quit: Leaving.)