00:01:06  * quijotequit (Ping timeout: 276 seconds)
00:04:21  * therealkoopaquit (Ping timeout: 276 seconds)
00:20:58  * notmattjoined
00:42:25  * fredkquit (Quit: Leaving.)
00:54:40  * nfitchquit (Quit: Leaving.)
00:56:36  * quijotejoined
00:57:18  * ryancnelsonquit (Quit: Leaving.)
00:59:36  * marsellquit (Quit: marsell)
01:01:04  * quijotequit (Ping timeout: 265 seconds)
01:02:44  * notmattquit (Remote host closed the connection)
01:03:20  * notmattjoined
01:07:35  * notmattquit (Ping timeout: 255 seconds)
01:13:26  * yunongquit (Ping timeout: 255 seconds)
01:20:00  * ed209quit (Remote host closed the connection)
01:20:07  * ed209joined
01:57:11  * quijotejoined
02:00:03  * trentmquit (Quit: Leaving.)
02:01:14  * trentmjoined
02:01:33  * ryancnelson1quit (Quit: Leaving.)
02:01:58  * quijotequit (Ping timeout: 265 seconds)
02:22:38  * trentmquit (Quit: Leaving.)
02:23:24  * notmattjoined
02:29:06  * AvianFluquit (Read error: Connection reset by peer)
02:33:49  * AvianFlujoined
02:38:55  * therealkoopajoined
02:43:31  * therealkoopaquit (Ping timeout: 252 seconds)
02:54:48  * notmattquit (Remote host closed the connection)
02:55:22  * notmattjoined
02:58:02  * quijotejoined
03:00:00  * notmatt_joined
03:00:17  * notmattquit (Ping timeout: 264 seconds)
03:03:06  * quijotequit (Ping timeout: 276 seconds)
03:04:25  * notmatt_quit (Ping timeout: 252 seconds)
03:06:06  * notmattjoined
03:10:17  * notmattquit (Ping timeout: 245 seconds)
03:14:12  * notmattjoined
03:19:56  * therealkoopajoined
03:21:30  * trentmjoined
03:24:37  * therealkoopaquit (Ping timeout: 265 seconds)
03:55:55  * trentmquit (Quit: Leaving.)
03:58:56  * quijotejoined
04:03:11  * quijotequit (Ping timeout: 252 seconds)
04:07:24  * trentmjoined
04:07:51  * trentmquit (Client Quit)
04:19:41  * therealkoopajoined
04:27:50  * therealkoopaquit (Ping timeout: 255 seconds)
04:49:02  * ryancnelsonjoined
04:52:36  * ed209quit (Remote host closed the connection)
04:52:43  * ed209joined
04:52:49  * ed209quit (Remote host closed the connection)
04:52:56  * ed209joined
04:53:10  <ryancnelson>.
04:59:30  * quijotejoined
05:04:10  * quijotequit (Ping timeout: 258 seconds)
05:19:40  * therealkoopajoined
05:24:02  * therealkoopaquit (Ping timeout: 245 seconds)
05:28:36  * ryancnelsonquit (Quit: Leaving.)
05:41:13  * therealkoopajoined
05:45:29  * therealkoopaquit (Ping timeout: 252 seconds)
06:00:16  * quijotejoined
06:04:55  * quijotequit (Ping timeout: 252 seconds)
06:10:43  * notmattquit (Remote host closed the connection)
06:11:16  * notmattjoined
06:15:33  * notmattquit (Ping timeout: 252 seconds)
06:19:50  * therealkoopajoined
06:24:17  * therealkoopaquit (Ping timeout: 264 seconds)
06:25:34  * therealkoopajoined
06:34:29  * therealkoopaquit (Ping timeout: 264 seconds)
06:36:44  * therealkoopajoined
06:42:09  * therealkoopaquit (Ping timeout: 276 seconds)
06:55:11  * AvianFluquit (Read error: Connection reset by peer)
06:55:49  * quijotejoined
06:55:54  * AvianFlujoined
07:05:06  * AvianFlu_joined
07:07:29  * AvianFluquit (Ping timeout: 264 seconds)
07:08:12  * notmattjoined
07:12:45  * notmattquit (Ping timeout: 252 seconds)
07:25:54  * AvianFlu_quit (Read error: Connection reset by peer)
07:26:53  * AvianFlujoined
07:37:24  * quijotequit (Ping timeout: 265 seconds)
08:07:21  * AvianFlu_joined
08:08:33  * AvianFluquit (Ping timeout: 258 seconds)
08:18:16  * AvianFlujoined
08:19:38  * AvianFlu_quit (Ping timeout: 240 seconds)
08:19:40  * therealkoopajoined
08:19:49  * quijotejoined
08:24:46  * therealkoopaquit (Ping timeout: 265 seconds)
08:27:05  * quijotequit (Ping timeout: 252 seconds)
08:31:44  * therealkoopajoined
08:35:41  * quijotejoined
08:35:58  * therealkoopaquit (Ping timeout: 240 seconds)
08:51:46  * AvianFluquit (Read error: Connection reset by peer)
08:52:42  * AvianFlujoined
08:56:08  * notmattjoined
09:00:59  * notmattquit (Ping timeout: 255 seconds)
09:01:04  * therealkoopajoined
09:03:27  * mamashjoined
09:06:27  * therealkoopaquit (Ping timeout: 276 seconds)
09:34:52  * quijotequit (Ping timeout: 245 seconds)
10:11:28  * notmattjoined
10:14:35  * therealkoopajoined
10:16:29  * notmattquit (Ping timeout: 264 seconds)
10:20:00  * ed209quit (Remote host closed the connection)
10:20:08  * ed209joined
10:23:07  * marselljoined
10:32:56  * quijotejoined
10:34:49  * quijote_joined
10:34:53  * quijotequit (Read error: Connection reset by peer)
10:39:35  * quijote_quit (Ping timeout: 258 seconds)
10:42:21  * therealkoopaquit (Remote host closed the connection)
10:47:24  * quijotejoined
11:27:37  * therealkoopajoined
11:31:32  * quijotequit (Ping timeout: 245 seconds)
11:32:38  * therealkoopaquit (Ping timeout: 255 seconds)
11:53:45  * quijotejoined
12:25:28  * quijotequit (Ping timeout: 265 seconds)
13:03:24  * quijotejoined
13:03:36  * therealkoopajoined
13:29:45  * quijotequit (Read error: Connection reset by peer)
13:30:18  * quijotejoined
13:39:55  * therealkoopaquit (Remote host closed the connection)
13:51:06  * therealkoopajoined
13:51:06  * mamashpart
13:56:11  * therealkoopaquit (Ping timeout: 255 seconds)
13:58:26  * therealkoopajoined
14:19:29  * therealkoopaquit (Remote host closed the connection)
14:20:18  * therealkoopajoined
14:21:49  * mamashjoined
14:54:17  * quijotequit (Ping timeout: 252 seconds)
15:01:47  * ryancnelsonjoined
15:05:19  * quijotejoined
15:08:11  * fredkjoined
15:09:25  * therealkoopaquit (Remote host closed the connection)
15:19:48  * nfitchjoined
15:27:56  * therealkoopajoined
15:37:03  * therealkoopaquit (Ping timeout: 258 seconds)
16:04:36  * therealkoopajoined
16:16:50  * trentmjoined
16:19:39  * yunongjoined
16:22:47  * quijotequit (Ping timeout: 265 seconds)
16:24:41  * notmattjoined
16:26:38  * trentmquit (Quit: Leaving.)
16:31:52  * trentmjoined
16:34:59  * quijotejoined
16:37:09  * RyanGordonjoined
16:40:24  <RyanGordon>Hello, I was talking to Richard from JPC support and he indicated that it might be possible to achieve parallel uploading to manta via something called "read streams"? I was wondering if anyone here knew if this was the case and if that could be elaborated on
16:40:58  <rmustacc>So how are you trying to upload things today?
16:43:04  * trentmquit (Quit: Leaving.)
16:43:16  <RyanGordon>Right now we're going through the Joyent API (using the mput/mget tools)
16:46:24  * mcavagejoined
16:46:51  <dap_>There's not really a great way to upload objects in parallel, except maybe to upload separate objects and then concatenate them with a job.
16:46:59  <dap_>You can download in parallel.
16:47:25  <rmustacc>Are you trying to upload one object in parallel, or you just have a lot of objects and you want to upload them in parallel?
16:48:10  <RyanGordon>Yeah, that was what I was told, but our use case is being able to upload a huge gzipped tarball (like 200GB) and traditionally we'd be able to use the HTTP byte offset headers for our tool to do the upload in parallel
16:50:10  * quijotequit (Read error: Connection reset by peer)
16:50:47  * quijotejoined
16:52:42  <dap_>The problem is that Manta doesn't support modifying existing objects; it's all-or-nothing.
16:53:19  <mcavage>RyanGordon: if it's that large, then dap's earlier suggestion would work; you don't have HTTP range headers on PUT, but you could do a split -> mput -> job to concat.
16:53:20  <dap_>If this is a one-time thing, I would consider chopping the file into N files, uploading those in parallel, and running a job that concatenates them all
16:53:35  * trentmjoined
16:53:52  <dap_>jinx!
16:57:21  <RyanGordon>Yeah, that's a possible solution but this is going to be a recurring task weekly or daily and timing is an issue so we're trying to make this as efficient as possible
16:57:51  * irajoined
16:58:35  <nfitch>RyanGordon: If you're at liberty to say, are you planning on using the resulting file in a job? If this is for backup purposes, you can always store it manta split (allowing you to upload in parallel) and reconstitute it as a whole only on download (when you need to restore).
16:59:12  <dap_>It's a question of whether the added concurrency is better than the cost of two passes, one of them very local. I think you'd have to try both to see. Also, what nfitch said.
16:59:12  <nfitch>*it in manta
17:01:09  <RyanGordon>Probably not going to use it in a job in manta after it's uploaded, but yes it would be for backups so we're just trying to keep the process as straight forward as possible so there's less risk of something going wrong. Splitting the files also makes it harder to get a consistent checksum of the uploaded file against the local file
17:01:46  <RyanGordon>Unless we split it, upload it, download it, recombine it and compare the two checksums which is something we're trying to avoid
17:02:23  <dap_>"download and recombine it" is the part we're suggesting doing in a Manta job
17:03:05  <nfitch>You could always compare the checksums of the split parts (if you split consistently), but, yes, understood that there's complexity there :/
17:04:10  <RyanGordon>Yeah, those are also both possible workarounds
17:11:31  * ryancnelsonquit (Ping timeout: 240 seconds)
17:15:00  * AvianFluquit (Remote host closed the connection)
17:16:18  * AvianFlujoined
17:18:21  * ryancnelsonjoined
17:21:48  * quijotequit (Ping timeout: 240 seconds)
17:28:18  * mamashpart
17:29:40  * quijotejoined
17:29:54  * ryancnelsonquit (Remote host closed the connection)
17:30:30  * ryancnelsonjoined
17:39:09  * ryancnelsonquit (Quit: Leaving.)
17:46:18  * quijotequit (Ping timeout: 240 seconds)
17:50:11  * AvianFluquit (Remote host closed the connection)
17:54:08  * yunongquit (Remote host closed the connection)
17:54:26  * yunongjoined
17:54:30  * yunongquit (Remote host closed the connection)
17:55:05  * yunongjoined
18:00:06  * yunongquit (Ping timeout: 276 seconds)
18:25:34  * fredkquit (Quit: Leaving.)
18:27:23  * fredkjoined
18:29:59  * quijotejoined
18:33:38  * yunongjoined
18:42:44  * AvianFlujoined
18:43:22  * AvianFluquit (Client Quit)
18:43:52  * AvianFlujoined
18:46:31  * notmattquit (Remote host closed the connection)
18:47:05  * notmattjoined
18:51:53  * notmattquit (Ping timeout: 264 seconds)
19:09:48  * quijotequit (Ping timeout: 258 seconds)
19:10:21  * ryancnelsonjoined
19:53:56  * quijotejoined
19:58:21  * quijotequit (Ping timeout: 252 seconds)
20:05:16  * iraquit (Quit: Connection terminated.)
20:20:00  * ed209quit (Remote host closed the connection)
20:20:07  * ed209joined
20:33:34  * dap_quit (Quit: Leaving.)
20:34:00  * dap_joined
20:54:49  * quijotejoined
20:58:59  * quijotequit (Ping timeout: 240 seconds)
21:31:47  * dap_quit (Quit: Leaving.)
21:34:25  * lloyddejoined
21:37:04  * dap_joined
21:55:25  * quijotejoined
22:00:09  * quijotequit (Ping timeout: 265 seconds)
22:07:44  * notmattjoined
22:18:52  * therealkoopaquit (Remote host closed the connection)
22:22:11  * therealkoopajoined
22:24:16  * therealkoopaquit (Remote host closed the connection)
22:24:32  * therealkoopajoined
22:33:20  * AvianFluquit (Remote host closed the connection)
22:44:45  * therealkoopaquit (Remote host closed the connection)
22:51:50  * therealkoopajoined
22:56:27  * quijotejoined
22:56:41  * therealkoopaquit (Ping timeout: 264 seconds)
22:58:09  * nfitchquit (Ping timeout: 265 seconds)
22:59:23  * nfitchjoined
23:00:11  * therealkoopajoined
23:00:51  * quijotequit (Ping timeout: 252 seconds)
23:04:24  * therealkoopaquit (Ping timeout: 258 seconds)
23:09:42  * RyanGordonquit (Quit: Lingo - http://lingoirc.com)
23:15:03  * lloyddequit (Remote host closed the connection)
23:45:30  * mcavagequit (Remote host closed the connection)
23:50:53  * irajoined
23:52:53  * ryancnelsonquit (Ping timeout: 255 seconds)
23:56:55  * yunongquit (Ping timeout: 258 seconds)
23:56:57  * therealkoopajoined
23:56:58  * quijotejoined
23:59:40  * aboutGodjoined