diff options
author | lioux <lioux@FreeBSD.org> | 2006-05-09 13:05:28 +0000 |
---|---|---|
committer | lioux <lioux@FreeBSD.org> | 2006-05-09 13:05:28 +0000 |
commit | 081dbb8e796b607149a52593529fbe4ab6669b73 (patch) | |
tree | e28c28d0ee644391e68fa8c33859f9f478a436d0 /net-p2p | |
parent | 8895f3b75a634ecf5b6a8567173f3aeed87630b3 (diff) | |
download | FreeBSD-ports-081dbb8e796b607149a52593529fbe4ab6669b73.zip FreeBSD-ports-081dbb8e796b607149a52593529fbe4ab6669b73.tar.gz |
o Update WIP swarming patch to revision h
o Fix: depend on gtk20 instead of gtk12
o Bump PORTREVISION
Diffstat (limited to 'net-p2p')
15 files changed, 2406 insertions, 352 deletions
diff --git a/net-p2p/mldonkey-devel/Makefile b/net-p2p/mldonkey-devel/Makefile index 482d442..a7a6291 100644 --- a/net-p2p/mldonkey-devel/Makefile +++ b/net-p2p/mldonkey-devel/Makefile @@ -7,7 +7,7 @@ PORTNAME= mldonkey PORTVERSION= 2.7.5 -PORTREVISION= 1 +PORTREVISION= 2 CATEGORIES+= net-p2p MASTER_SITES= ${MASTER_SITE_SOURCEFORGE_EXTENDED} \ ${MASTER_SITE_SAVANNAH} @@ -74,7 +74,7 @@ LIB_DEPENDS+= gd.4:${PORTSDIR}/graphics/gd \ CONFIGURE_ARGS+=--enable-gui=newgui2 # we don't need lablgtk as RUN dependency, but we need gtk+glib -USE_GNOME= gtk12 +USE_GNOME= gtk20 MAKE_ENV+= OCAMLRUNPARAM="l=256M" .if defined(WITHOUT_CORE) @@ -144,7 +144,7 @@ post-patch: .endfor pre-configure: - cd ${WRKSRC}/config && ${AUTOCONF} + @cd ${WRKSRC}/config && ${AUTOCONF} post-configure: @${REINPLACE_CMD} -E \ diff --git a/net-p2p/mldonkey-devel/files/patch-src__daemon__common__commonSwarming.ml b/net-p2p/mldonkey-devel/files/patch-src__daemon__common__commonSwarming.ml index 4abf091..d02ff34 100644 --- a/net-p2p/mldonkey-devel/files/patch-src__daemon__common__commonSwarming.ml +++ b/net-p2p/mldonkey-devel/files/patch-src__daemon__common__commonSwarming.ml @@ -1,15 +1,55 @@ --- ./src/daemon/common/commonSwarming.ml.orig Mon Apr 10 14:06:20 2006 -+++ ./src/daemon/common/commonSwarming.ml Thu Apr 20 11:04:03 2006 -@@ -38,7 +38,7 @@ ++++ ./src/daemon/common/commonSwarming.ml Sun May 7 06:39:10 2006 +@@ -17,6 +17,26 @@ + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + *) + ++(* OVERALL SCHEMA ++ ++Each network frontend can have a different (fixed) chunk size ++t1 +--------+--------+--------+--------+--------+--------+-------- chunks ++t2 +------+------+------+------+------+------+------+------+------ chunks ++ ++ each block is contained in at most /\ chunk_of_block ++ one chunk, for any network || mappings ++ \/ blocks_of_chunk ++swarmer ++ +------+-+----+---+--+------+------++-----+--+---+----+-+------ blocks ++ | | | | | | | ... variable size ++ v v v v v v v ++ r<>r r r r r r<>r r<>r ranges ++ ^ one dbl linked list/block ++ | encoding missing data ranges ++uploaders physically uploader ++reference ranges ++*) ++ + open Int64ops + open Options + open Printf2 +@@ -25,7 +45,6 @@ + + + let check_swarming = false +-let debug_present_chunks = false + let debug_all = false + + open CommonTypes +@@ -37,11 +56,8 @@ + exception VerifierNotReady - type chunks = +-type chunks = - AvailableRanges of (int64 * int64) list +-(* A bitmap is encoded with '0' for empty, '1' for present '2' complete '3' verified *) +-| AvailableCharBitmap of string +-(* A bitmap encoded as a bit vector *) ++type intervals = + AvailableIntervals of (int64 * int64) list - (* A bitmap is encoded with '0' for empty, '1' for present '2' complete '3' verified *) - | AvailableCharBitmap of string - (* A bitmap encoded as a bit vector *) -@@ -54,18 +54,17 @@ + | AvailableBitv of Bitv.t + + type verification = +@@ -54,18 +70,17 @@ let exit_on_error = ref false (* prints a new logline with date, module and starts newline *) @@ -34,7 +74,7 @@ open CommonFile open CommonTypes open CommonClient -@@ -93,6 +92,8 @@ +@@ -93,6 +108,8 @@ *) @@ -43,7 +83,7 @@ type chunk = { chunk_uid : uid_type; chunk_size : int64; -@@ -102,20 +103,27 @@ +@@ -102,20 +119,27 @@ (* glossary: network frontend use "chunks" of data, swarmer use "blocks" of data *) @@ -76,7 +116,7 @@ (* mapping from network chunks to swarmer blocks *) mutable t_blocks_of_chunk : int list array; -@@ -124,12 +132,12 @@ +@@ -124,12 +148,12 @@ } and swarmer = { @@ -92,7 +132,7 @@ mutable s_range_size : int64; mutable s_strategy : strategy; -@@ -156,8 +164,11 @@ +@@ -156,20 +180,21 @@ mutable block_end : Int64.t; mutable block_ranges : range; (** [range] of the double-linked list of ranges associated to the @@ -106,10 +146,13 @@ } and range = { -@@ -167,9 +178,8 @@ + mutable range_block : block; +- mutable range_begin : Int64.t; (* official begin int64 *) ++ mutable range_begin : Int64.t; + mutable range_end : Int64.t; mutable range_prev : range option; mutable range_next : range option; - mutable range_current_begin : Int64.t; (* current begin pos *) +- mutable range_current_begin : Int64.t; (* current begin pos *) -(* mutable range_verified : bool; *) - mutable range_nuploading : int; (* current number of clients - filling that range ? *) @@ -118,7 +161,16 @@ } and uploader = { -@@ -193,11 +203,14 @@ +@@ -178,7 +203,7 @@ + + mutable up_declared : bool; + +- mutable up_chunks : chunks; ++ mutable up_intervals : intervals; + mutable up_complete_blocks : int array; (** block numbers *) + mutable up_ncomplete : int; + +@@ -193,11 +218,14 @@ mutable up_block_begin : int64; mutable up_block_end : int64; @@ -131,22 +183,22 @@ (* range invariants: - Ranges represent "holes" of missing data in a block. + Ranges represent "holes" of missing data in a block; Data is -+ missing between offsets range_current_begin and range_end. ++ missing between offsets range_begin and range_end. [block]'s [block_ranges] reference the first (smallest offsets) of the [range]s associated with it. -@@ -218,10 +231,45 @@ +@@ -216,12 +244,59 @@ + overlap, and are sorted in increasing offsets order: + b.block_begin <= b.block_ranges.block_begin ... <= - r.range_prev.range_end <= r.range_begin <= r.range_current_begin <= +- r.range_prev.range_end <= r.range_begin <= r.range_current_begin <= ++ r.range_prev.range_end <= r.range_begin <= r.range_end <= r.range_next.range_begin <= ... - <= b.block_end + <= b.block_end *) - Role played by r.range_current_begin is unclear for now. *) -+(* Role played by r.range_begin is unclear for now. One beginning -+ offset is probably enough for an interval ;) - -+ range owners are only used thru uploaders.up_ranges. blocks could be ++(* range owners are only used thru uploaders.up_ranges. blocks could be + saved in [uploaders]' [up_ranges] along range, but would + need uploading when the swarmer is splitted. + @@ -159,7 +211,7 @@ +(* block invariants + Data missing for a block is the sum of the "sizes" of its ranges. + -+ b.block_remaining = sum (r.range_end - r.range_current_begin) b.block_ranges ++ b.block_remaining = sum (r.range_end - r.range_begin) b.block_ranges +*) + +(* swarmer invariants ? @@ -167,7 +219,7 @@ + s.s_verified_bitmap.[i] = 1 <=> s_blocks.[i] = PartialBlock _ + s.s_verified_bitmap.[i] = 2 <=> s_blocks.[i] = CompletedBlock + s.s_verified_bitmap.[i] = 3 <=> s_blocks.[i] = VerifiedBlock -+ If so, why not drop s_verified_bitmap, and replace it by ++ If so, why not drop s_verified_bitmap, and replace it by some + verified_bitmap s i and verified_bitmap_all s functions ? +*) + @@ -177,13 +229,28 @@ + t_nverified_chunks = + List.length (List.filter (fun x -> x = '3') t_converted_verified_bitmap) + ++ hence t_ncomplete_chunks >= t_nverified_chunks ++ + All chunks are [t_chunk_size] bytes in size, and first start at + offset 0. This is assumed in [associate], [verify_chunk], maybe more. +*) ++ ++(* uploaders invariants ? ++ At first, I supposed ++ up_ncomplete = Array.length up_complete_blocks ++ up.up_npartial = Array.length up_partial_blocks ++ if so, since Array.length complexity is O(1), why keep them ? + ++ => see permute_and_return, they're used to simulate to removal of ++ elements without array reallocation ++ ++ So the question is now, aren't there better datastructures than ++ arrays for the job ? ++*) (*************************************************************************) (* *) -@@ -251,8 +299,6 @@ +@@ -251,8 +326,6 @@ let swarmer_counter = ref 0 @@ -192,7 +259,7 @@ (*************************************************************************) (* *) (* dummy_swarmer *) -@@ -276,10 +322,10 @@ +@@ -276,10 +349,10 @@ (** (debug) output an [uploader] to current log *) let print_uploader up = @@ -205,21 +272,52 @@ Array.iter (fun (i, begin_pos, end_pos) -> lprintf " %d[%Ld...%Ld] " i begin_pos end_pos ) up.up_partial_blocks; -@@ -301,10 +347,10 @@ +@@ -289,23 +362,37 @@ + associated file's [t.t_file] last seen value to the oldest of the + remaining last seen values *) + ++let string_for_all p s = ++ let l = String.length s in ++ let rec aux i = ++ i >= l || p s.[i] && aux (i+1) in ++ aux 0 ++ ++let string_iter f s = ++ let l = String.length s in ++ let rec aux i = ++ if i < l then begin ++ f i s.[i]; ++ aux (i+1) ++ end in ++ aux 0 ++ + let compute_last_seen t = + let last_seen_total = ref (BasicSocket.last_time ()) in +- for i = 0 to String.length t.t_converted_verified_bitmap - 1 do +- if t.t_converted_verified_bitmap.[i] > '2' then ++ string_iter (fun i c -> ++ if c > '2' then + t.t_last_seen.(i) <- BasicSocket.last_time () + else + last_seen_total := min !last_seen_total t.t_last_seen.(i) +- done; ++ ) t.t_converted_verified_bitmap; + set_file_last_seen t.t_file !last_seen_total; t.t_last_seen (** if a swarmer is already associated with that [file_name], return it; - Otherwise create a new one with default values (including a default - [range_size] instead of the provided value ??) *) +- +-let create_swarmer file_name file_size range_size = + Otherwise create a new one with default values, that will be fixed + by the first frontend association *) --let create_swarmer file_name file_size range_size = +let create_swarmer file_name file_size = - try HS.find swarmers_by_name -@@ -314,9 +360,7 @@ + { dummy_swarmer with +@@ -314,9 +401,7 @@ with Not_found -> incr swarmer_counter; @@ -230,12 +328,12 @@ let nchunks = 1 in let rec s = { -@@ -357,6 +401,43 @@ +@@ -357,6 +442,43 @@ let b = s.s_block_pos in b.(i) +(** Finds the number of the block containing [chunk_pos] offset, using -+ dichotomy *) ++ dichotomy. Blocks are half opened [block_begin, block_end[ *) + +(* 0 <= chunk_pos < s.s_size *) +let compute_block_num s chunk_pos = @@ -274,8 +372,74 @@ (** (internal) return a 0 sized range at offset [pos], and assigned to block [b] *) -@@ -426,7 +507,7 @@ - r.range_current_begin <- min r.range_current_begin cut_pos; +@@ -368,19 +490,40 @@ + range_end = pos; + range_block = b; + range_nuploading = 0; +- range_current_begin = pos; + } + in + r + ++let rec ranges_iter f r = ++ f r; ++ match r.range_next with ++ | None -> () ++ | Some r -> ++ ranges_iter f r ++ ++let rec ranges_fold f acc r = ++ let acc = f acc r in ++ match r.range_next with ++ | None -> acc ++ | Some rr -> ranges_fold f acc rr ++ ++let rec ranges_for_all p r = ++ p r && ++ (match r.range_next with ++ | None -> true ++ | Some r -> ranges_for_all p r) ++ ++let block_ranges_for_all p b = ++ ranges_for_all p b.block_ranges ++ ++let block_ranges_fold f acc b = ++ ranges_fold f acc b.block_ranges ++ + (** (internal) assigns range [r], and all other ranges along + [range_next] links, to block [b] *) + + let rec own_ranges b r = +- r.range_block <- b; +- match r.range_next with +- None -> () +- | Some r -> own_ranges b r ++ ranges_iter (fun r -> r.range_block <- b) r + + (** (internal) + Find ranges that are after [cut_pos] offset, unlink them from r +@@ -392,8 +535,6 @@ + If [cut_pos] is within one of the ranges, that range is cut in + two at [cut_pos] offset, and link each half to its side. + +- What should happen to range_begin is unclear. +- + Also, what do to if range_nuploaders is not 0 ? + => [cut_ranges_after] is being called from [split_blocks] that + does not preserve [s_nuploading] for blocks either +@@ -417,16 +558,15 @@ + (* "right" half *) + let split_r = { r with + range_prev = None; +- range_begin = cut_pos; +- range_current_begin = max r.range_current_begin cut_pos ++ range_begin = max r.range_begin cut_pos + } in + (* "left" half *) + r.range_next <- None; + r.range_end <- cut_pos; +- r.range_current_begin <- min r.range_current_begin cut_pos; ++ r.range_begin <- min r.range_begin cut_pos; if r.range_nuploading <> 0 then - lprintf_n () "WARNING: Splitting a range currently being uploaded, don't know what to do with range_nuploaders :/\n"; @@ -283,7 +447,51 @@ split_r in let cut_ranges = iter r in -@@ -554,9 +635,9 @@ +@@ -438,10 +578,10 @@ + + let empty_block b = + let rec iter begin_pos r = +- r.range_current_begin = begin_pos && ++ r.range_begin = begin_pos && + (match r.range_next with +- Some rr -> iter r.range_end rr +- | None -> r.range_end = b.block_end) ++ | Some rr -> iter r.range_end rr ++ | None -> r.range_end = b.block_end) + in + iter b.block_begin b.block_ranges + +@@ -490,7 +630,7 @@ + (* We need to split this block in two parts *) + s.s_block_pos.(index_s) <- chunk_end; + match s.s_blocks.(index_s) with +- EmptyBlock | CompleteBlock | VerifiedBlock -> ++ | EmptyBlock | CompleteBlock | VerifiedBlock -> + + (* s.s_blocks.(index_s) will appear twice in the result list *) + let new_blocks = ( +@@ -501,7 +641,6 @@ + iter index_s chunk_end new_blocks + + | PartialBlock b1 -> +- + (* split b1 in two; b2 is the part after [chunk_end] offset *) + let b2 = { + block_s = s; +@@ -535,12 +674,8 @@ + s.s_verified_bitmap.[index_s] <- '0'; + end else + s.s_blocks.(index_s) <- PartialBlock b2; +- + iter index_s chunk_end new_blocks +- + end +- +- + in + let blocks = iter 0 zero [] in + +@@ -554,9 +689,9 @@ aux 0 in if array_exist ((<>) 0) s.s_availability then @@ -295,7 +503,29 @@ s.s_blocks <- Array.create nblocks EmptyBlock; s.s_verified_bitmap <- String.make nblocks '0'; -@@ -617,7 +698,7 @@ +@@ -571,8 +706,8 @@ + | (b, pos, c) :: tail -> + begin + match b with +- PartialBlock b -> b.block_num <- i +- | _ -> () ++ | PartialBlock b -> b.block_num <- i ++ | EmptyBlock | CompleteBlock | VerifiedBlock -> () + end; + s.s_blocks.(i) <- b; + s.s_verified_bitmap.[i] <- c; +@@ -613,11 +748,18 @@ + (* TODO: transfer data into swarmer instead of discarding it *) + Unix32.remove (file_fd t.t_file); + end; ++ ++ (match s.s_networks with ++ | t :: tail -> ++ assert(t.t_primary); ++ List.iter (fun tt -> assert(not tt.t_primary)) tail ++ | [] -> assert false); ++ + (* at this point, we are supposed to split the blocks in the swarmer in smaller blocks depending on the block_size of this network, and compute the t_chunk_of_block and t_blocks_of_chunk fields. *) @@ -304,7 +534,7 @@ split_blocks s chunk_size; -@@ -628,7 +709,7 @@ +@@ -628,7 +770,7 @@ t.t_chunk_of_block <- Array.create nblocks 0; t.t_blocks_of_chunk <- Array.create nchunks []; @@ -313,7 +543,35 @@ for i = 0 to nblocks - 1 do let block_begin = compute_block_begin s i in let chunk = Int64.to_int (block_begin // chunk_size) in -@@ -663,21 +744,21 @@ +@@ -642,19 +784,14 @@ + add_file_downloaded t.t_file (zero -- file_downloaded t.t_file); + + (* check that all frontends use the primary's file backend *) +- begin +- match s.s_networks with +- t :: tail when is_primary -> +- List.iter (fun tt -> +- assert (not tt.t_primary); +- set_file_fd tt.t_file (file_fd t.t_file) +- ) tail +- +- | tt :: tail when tt.t_primary -> +- assert (not is_primary); +- set_file_fd t.t_file (file_fd tt.t_file) +- | _ -> () +- end; ++ (match s.s_networks with ++ | t :: tail when is_primary -> ++ List.iter (fun tt -> ++ set_file_fd tt.t_file (file_fd t.t_file) ++ ) tail ++ | tprim :: tail -> ++ set_file_fd t.t_file (file_fd tprim.t_file) ++ | [] -> assert false); + + () + +@@ -663,21 +800,21 @@ let create ss file chunk_size = let size = file_size file in @@ -340,14 +598,43 @@ t_converted_verified_bitmap = String.make nchunks '0'; t_last_seen = Array.create nchunks 0; -@@ -721,91 +802,59 @@ - let s = t.t_s in - s.s_nuploading.(num) <- s.s_nuploading.(num) - 1 +@@ -692,120 +829,59 @@ + associate true t ss; + t --(** Finds the number of the block containing [chunk_pos] offset, using -- dichotomy *) +(** iter function f over all the blocks contained in the list of [intervals] +-(*************************************************************************) +-(* *) +-(* clear_uploader_ranges *) +-(* *) +-(*************************************************************************) +- +-let clear_uploader_ranges up = +- List.iter (fun (_,_,r) -> +- r.range_nuploading <- r.range_nuploading - 1 +- ) up.up_ranges; +- up.up_ranges <- [] +- +-(*************************************************************************) +-(* *) +-(* clear_uploader_block *) +-(* *) +-(*************************************************************************) +- +-let clear_uploader_block up = +- match up.up_block with +- None -> () +- | Some b -> +- up.up_block <- None; +- let num = b.block_num in +- let t = up.up_t in +- let s = t.t_s in +- s.s_nuploading.(num) <- s.s_nuploading.(num) - 1 +- +-(** Finds the number of the block containing [chunk_pos] offset, using +- dichotomy *) +- -let compute_block_num s chunk_pos = - let b = s.s_block_pos in - let rec iter min max = @@ -456,8 +743,9 @@ let rec iter r = - lprintf_n () " %Ld(%Ld)-%Ld(%d)" -+ lprintf_n " %Ld(%Ld)-%Ld(%d)" - r.range_begin r.range_current_begin r.range_end r.range_nuploading; +- r.range_begin r.range_current_begin r.range_end r.range_nuploading; ++ lprintf_n " %Ld-%Ld(%d)" ++ r.range_begin r.range_end r.range_nuploading; match r.range_next with None -> lprint_newline() - | Some r -> iter r @@ -470,7 +758,7 @@ let block_begin = compute_block_begin s i in let block_end = compute_block_end s i in lprintf "%Ld - %Ld [%Ld] %c " block_begin block_end -@@ -818,7 +867,7 @@ +@@ -818,7 +894,7 @@ ) s.s_networks; match b with @@ -479,7 +767,7 @@ lprintf " [%Ld .. %Ld] --> " b.block_begin b.block_end; iter b.block_ranges -@@ -827,19 +876,15 @@ +@@ -827,119 +903,70 @@ | VerifiedBlock -> lprintf_nl2 "V" ) s.s_blocks; @@ -505,7 +793,13 @@ let iter_block_ranges f b = let rec iter_range f r = -@@ -850,96 +895,50 @@ ++ let next = r.range_next in (* keep next range in case f mutates it *) + f r; +- match r.range_next with +- None -> () ++ match next with ++ | None -> () + | Some rr -> iter_range f rr in iter_range f b.block_ranges @@ -531,7 +825,7 @@ - iter_range b.block_ranges; + lprintf_nl " ranges:"; + iter_block_ranges (fun r -> -+ lprintf_nl " %Ld-%Ld" r.range_current_begin r.range_end) b; ++ lprintf_nl " %Ld-%Ld" r.range_begin r.range_end) b; lprint_newline () -(*************************************************************************) @@ -621,10 +915,10 @@ - | _ -> () +let close_block_ranges maybe_t s b = + iter_block_ranges (fun r -> -+ let added = r.range_end -- r.range_current_begin in ++ let added = r.range_end -- r.range_begin in + add_file_downloaded maybe_t s added; + b.block_remaining <- b.block_remaining -- added; -+ r.range_current_begin <- r.range_end; ++ r.range_begin <- r.range_end; + r.range_prev <- None; + r.range_next <- None) b; + if b.block_remaining <> 0L then @@ -632,7 +926,7 @@ (*************************************************************************) (* *) -@@ -947,146 +946,243 @@ +@@ -947,146 +974,245 @@ (* *) (*************************************************************************) @@ -683,32 +977,15 @@ - if s.s_verified_bitmap.[i] = '0' then begin - s.s_verified_bitmap.[i] <- '1'; - List.iter (fun t -> -- let j = t.t_chunk_of_block.(i) in -- if t.t_converted_verified_bitmap.[j] = '0' then -- t.t_converted_verified_bitmap.[j] <- '1' -- ) s.s_networks -- end -- --(* we finished this block, we need know to verify it *) --let set_bitmap_2 s i = -- if s.s_verified_bitmap.[i] < '2' then begin -- s.s_verified_bitmap.[i] <- '2'; -- match s.s_networks with -- | t :: _ when t.t_primary -> +let set_swarmer_bitmap_1 s i = + match s.s_verified_bitmap.[i] with + | '0' -> + s.s_verified_bitmap.[i] <- '1'; + List.iter (fun t -> let j = t.t_chunk_of_block.(i) in -- if List.for_all (fun i -> s.s_verified_bitmap.[i] = '2') -- t.t_blocks_of_chunk.(j) && -- t.t_converted_verified_bitmap.[j] <> '3' then begin -- t.t_ncomplete_blocks <- t.t_ncomplete_blocks + 1; -- t.t_converted_verified_bitmap.[j] <- '2' -- end -- | [] -> assert false -- | _ -> () +- if t.t_converted_verified_bitmap.[j] = '0' then +- t.t_converted_verified_bitmap.[j] <- '1' +- ) s.s_networks - end + match t.t_converted_verified_bitmap.[j] with + | '0' -> t.t_converted_verified_bitmap.[j] <- '1' @@ -723,16 +1000,22 @@ + | _ -> assert false + --(* the primary verifier has worked, so let ask secondary ones for --verification too *) --let set_bitmap_3 s i = -- if s.s_verified_bitmap.[i] < '3' then begin -- s.s_verified_bitmap.[i] <- '3'; --(* lprintf "set_bitmap_3 %d done\n" i; *) +-(* we finished this block, we need know to verify it *) +-let set_bitmap_2 s i = +- if s.s_verified_bitmap.[i] < '2' then begin +- s.s_verified_bitmap.[i] <- '2'; - match s.s_networks with -- [] -> assert false -- | tprim :: tail -> -- List.iter (fun t -> +- | t :: _ when t.t_primary -> +- let j = t.t_chunk_of_block.(i) in +- if List.for_all (fun i -> s.s_verified_bitmap.[i] = '2') +- t.t_blocks_of_chunk.(j) && +- t.t_converted_verified_bitmap.[j] <> '3' then begin +- t.t_ncomplete_blocks <- t.t_ncomplete_blocks + 1; +- t.t_converted_verified_bitmap.[j] <- '2' +- end +- | [] -> assert false +- | _ -> () +- end +(* we finished this block, trying to escalate to primary frontend + verification bitmap *) +let set_swarmer_bitmap_2 s i = @@ -742,17 +1025,7 @@ + match s.s_networks with + | t :: _ -> + assert (t.t_primary); - let j = t.t_chunk_of_block.(i) in -- if List.for_all (fun i -> s.s_verified_bitmap.[i] = '3') -- t.t_blocks_of_chunk.(j) then -- match t.t_verifier with -- NoVerification -> -- t.t_converted_verified_bitmap.[j] <- '3' -- | _ -> -- t.t_ncomplete_blocks <- t.t_ncomplete_blocks + 1; -- t.t_converted_verified_bitmap.[j] <- '2' -- ) tail -- end ++ let j = t.t_chunk_of_block.(i) in + (match t.t_converted_verified_bitmap.[j] with + | '0' | '1' -> + if List.for_all (fun i -> s.s_verified_bitmap.[i] = '2') @@ -768,65 +1041,88 @@ + | '3' -> lprintf_nl "set_swarmer_bitmap_2: trying to demote a verified block? (2)" + | _ -> assert false + (* the primary verifier has worked, so let ask secondary ones for +-verification too *) +-let set_bitmap_3 s i = +- if s.s_verified_bitmap.[i] < '3' then begin +- s.s_verified_bitmap.[i] <- '3'; +-(* lprintf "set_bitmap_3 %d done\n" i; *) ++ verification too *) ++let set_swarmer_bitmap_3 s i = ++ match s.s_verified_bitmap.[i] with ++ | '0' | '1' | '2' -> ++ (s.s_verified_bitmap.[i] <- '3'; ++(* lprintf "set_swarmer_bitmap_3 %d done\n" i; *) + match s.s_networks with +- [] -> assert false +- | tprim :: tail -> +- List.iter (fun t -> +- let j = t.t_chunk_of_block.(i) in +- if List.for_all (fun i -> s.s_verified_bitmap.[i] = '3') +- t.t_blocks_of_chunk.(j) then +- match t.t_verifier with +- NoVerification -> +- t.t_converted_verified_bitmap.[j] <- '3' +- | _ -> +- t.t_ncomplete_blocks <- t.t_ncomplete_blocks + 1; +- t.t_converted_verified_bitmap.[j] <- '2' +- ) tail +- end +- -(*************************************************************************) -(* *) -(* set_toverify_block (internal) *) -(* *) -(*************************************************************************) -+(* the primary verifier has worked, so let ask secondary ones for -+ verification too *) -+let set_swarmer_bitmap_3 s i = -+ match s.s_verified_bitmap.[i] with -+ | '0' | '1' | '2' -> -+ (s.s_verified_bitmap.[i] <- '3'; -+(* lprintf "set_swarmer_bitmap_3 %d done\n" i; *) -+ match s.s_networks with -+ | tprim :: secondaries -> -+ assert (tprim.t_primary); +- +- (* +-let set_toverify_block s i = set_bitmap_2 s i +- *) ++ | [] -> assert false ++ | tprim :: secondaries -> ++ assert (tprim.t_primary); ++ match tprim.t_verifier with ++ | NoVerification | VerificationNotAvailable -> () ++ | Verification _ | ForceVerification -> + let jprim = tprim.t_chunk_of_block.(i) in + assert (tprim.t_converted_verified_bitmap.[jprim] = '3'); -+ List.iter (fun t -> ++ List.iter (fun t -> + assert (not t.t_primary); + let j = t.t_chunk_of_block.(i) in + if List.for_all (fun i -> s.s_verified_bitmap.[i] = '3') + t.t_blocks_of_chunk.(j) then -+ match t.t_verifier with -+ | NoVerification -> -+ (* we have no way to check data integrity -+ for this network, assume other(s) know -+ better *) -+ (match t.t_converted_verified_bitmap.[j] with -+ | '0' | '1' -> -+ t.t_converted_verified_bitmap.[j] <- '3'; -+ t.t_ncomplete_chunks <- t.t_ncomplete_chunks + 1; -+ t.t_nverified_chunks <- t.t_nverified_chunks + 1 -+ | '2' -> -+ t.t_converted_verified_bitmap.[j] <- '3'; -+ t.t_nverified_chunks <- t.t_nverified_chunks + 1 -+ | '3' -> () -+ | _ -> assert false) -+ | VerificationNotAvailable -+ | ForceVerification -+ | Verification _ -> -+ (* all chunks are verified, so set -+ converted_verified_bitmap to '2', -+ probably to trigger data verification later. - -- (* --let set_toverify_block s i = set_bitmap_2 s i -- *) -+ Is that code necessary at all ? *) -+ (match t.t_converted_verified_bitmap.[j] with -+ | '0' | '1' -> -+ t.t_converted_verified_bitmap.[j] <- '2'; -+ t.t_ncomplete_chunks <- t.t_ncomplete_chunks + 1 -+ | '2' -> () -+ | '3' -> lprintf_nl "set_swarmer_bitmap_3: trying to demote a verified block in another frontend?" -+ | _ -> assert false) -+ ) secondaries -+ | [] -> assert false) -+ | '3' -> () -+ | _ -> assert false ++ match t.t_verifier with ++ | NoVerification | VerificationNotAvailable -> ++ (* we have no way to check data integrity ++ for this network, assume other(s) know ++ better *) ++ (match t.t_converted_verified_bitmap.[j] with ++ | '0' | '1' -> ++ t.t_converted_verified_bitmap.[j] <- '3'; ++ t.t_ncomplete_chunks <- t.t_ncomplete_chunks + 1; ++ t.t_nverified_chunks <- t.t_nverified_chunks + 1 ++ | '2' -> ++ t.t_converted_verified_bitmap.[j] <- '3'; ++ t.t_nverified_chunks <- t.t_nverified_chunks + 1 ++ | '3' -> () ++ | _ -> assert false) ++ | ForceVerification ++ | Verification _ -> ++ (* all chunks are verified, so set ++ converted_verified_bitmap to '2', ++ probably to trigger data verification later. ++ ++ Is that code necessary at all ? *) ++ (match t.t_converted_verified_bitmap.[j] with ++ | '0' | '1' -> ++ t.t_converted_verified_bitmap.[j] <- '2'; ++ t.t_ncomplete_chunks <- t.t_ncomplete_chunks + 1 ++ | '2' -> () ++ | '3' -> lprintf_nl "set_swarmer_bitmap_3: trying to demote a verified block in another frontend?" ++ | _ -> assert false) ++ ) secondaries) ++ | '3' -> () ++ | _ -> assert false -(*************************************************************************) -(* *) @@ -984,28 +1280,13 @@ (*************************************************************************) (* *) -@@ -1094,163 +1190,62 @@ +@@ -1094,258 +1220,107 @@ (* *) (*************************************************************************) -let verify_chunk t i = - if t.t_converted_verified_bitmap.[i] = '2' then - let nblocks = String.length t.t_converted_verified_bitmap in -+let string_for_all p s = -+ let l = String.length s in -+ let rec aux i = -+ i >= l || p s.[i] && aux (i+1) in -+ aux 0 -+ -+let string_iter f s = -+ let l = String.length s in -+ let rec aux i = -+ if i < l then begin -+ f i s.[i]; -+ aux (i+1) -+ end in -+ aux 0 -+ +let verify_chunk t j = + if t.t_converted_verified_bitmap.[j] = '2' then + let nchunks = String.length t.t_converted_verified_bitmap in @@ -1190,28 +1471,168 @@ - verify_chunk t i - *) - (*************************************************************************) - (* *) -@@ -1259,7 +1254,7 @@ - (*************************************************************************) +-(*************************************************************************) +-(* *) +-(* must_verify_block *) +-(* *) +-(*************************************************************************) ++(** mark a block as completed, ready for verification *) - let must_verify_block s i immediatly = +-let must_verify_block s i immediatly = - set_bitmap_2 s i; -+ set_swarmer_bitmap_2 s i; - if immediatly then - match s.s_networks with - [] -> assert false -@@ -1296,7 +1291,7 @@ +- if immediatly then +- match s.s_networks with +- [] -> assert false +- | t :: _ when t.t_primary -> +- let i = t.t_chunk_of_block.(i) in +- t.t_converted_verified_bitmap.[i] <- '2'; +-(* List.iter (fun j -> +- if s.s_verified_bitmap.[j] <> '2' then begin +- lprintf " block %d not downloaded\n" j; +- exit_on_error := false; +- end; +- ) t.t_blocks_of_chunk.(i); *) +- verify_chunk t i; +-(* exit_on_error := true; *) +- | _ -> () ++let must_verify_block s i = ++ set_swarmer_bitmap_2 s i + +-(*************************************************************************) +-(* *) +-(* verify_all_blocks *) +-(* *) +-(*************************************************************************) ++(** mark all blocks as completed, ready for verification *) + +-let verify_all_chunks t immediatly = ++let verify_all_chunks t = + let s = t.t_s in +- for i = 0 to String.length s.s_verified_bitmap - 1 do +- must_verify_block s i immediatly +- done ++ string_iter (fun i _ -> must_verify_block s i) s.s_verified_bitmap + +-(*************************************************************************) +-(* *) +-(* compute_bitmap *) +-(* *) +-(*************************************************************************) ++(** same, and synchronously calls the verification of all chunks *) ++ ++let verify_all_chunks_immediately t = ++ verify_all_chunks t; ++ string_iter (fun i _ -> verify_chunk t i) t.t_converted_verified_bitmap ++ ++(** synchronously verify all completed chunks not yet verified *) let compute_bitmap t = - if t.t_ncomplete_blocks > t.t_nverified_blocks then begin -+ if t.t_ncomplete_chunks > t.t_nverified_chunks then begin - for i = 0 to String.length t.t_converted_verified_bitmap - 1 do - if t.t_converted_verified_bitmap.[i] = '2' then - verify_chunk t i -@@ -1384,38 +1379,10 @@ +- for i = 0 to String.length t.t_converted_verified_bitmap - 1 do +- if t.t_converted_verified_bitmap.[i] = '2' then +- verify_chunk t i +- done +- end ++ if t.t_ncomplete_chunks > t.t_nverified_chunks then ++ string_iter (fun i c -> ++ if c = '2' then verify_chunk t i) t.t_converted_verified_bitmap + + +-(*************************************************************************) +-(* *) +-(* split_range (internal) *) +-(* *) +-(*************************************************************************) ++(** Replaces the ith block of the swarmer with a PartialBlock ++ ranges are created with s_range_size size *) + +-let rec split_range r range_size = +- assert (r.range_current_begin = r.range_begin); +- let next_range = r.range_begin ++ range_size in +-(* lprintf " split_range: next_range %Ld\n" next_range; *) +- if r.range_end > next_range then +- let rr = { +- range_block = r.range_block; +- range_nuploading = 0; +- range_next = r.range_next; +- range_prev = Some r; +- range_begin = next_range; +- range_current_begin = next_range; +- range_end = r.range_end; ++let new_block s i = ++ ++ (** Split a range in ranges of at most [range_size] bytes, if needed ++ ranges stay linked together *) ++ ++ let rec split_range r range_size = ++ let cut_pos = r.range_begin ++ range_size in ++(* lprintf " split_range: cut_pos %Ld\n" cut_pos; *) ++ if r.range_end > cut_pos then ++ (* "right" half *) ++ let split_r = { ++ range_block = r.range_block; ++ range_nuploading = 0; ++ range_next = r.range_next; ++ range_prev = Some r; ++ range_begin = cut_pos; ++ range_end = r.range_end; + } in +- begin +- match r.range_next with +- None -> () +- | Some rrr -> ++ (match r.range_next with ++ | None -> () ++ | Some old_next_range -> + (* lprintf "Another one ??\n"; *) +- rrr.range_prev <- Some rr; +- end; +- r.range_next <- Some rr; +- r.range_end <- next_range; ++ old_next_range.range_prev <- Some split_r); ++ (* "left" half *) ++ r.range_next <- Some split_r; ++ r.range_end <- cut_pos; + (* lprintf " NEW RANGE: %Ld- OLD RANGE: %Ld-%Ld\n" +- rr.range_begin r.range_begin r.range_end; *) +- +- split_range rr range_size +- +- +-(*************************************************************************) +-(* *) +-(* new_block (internal) *) +-(* *) +-(*************************************************************************) ++ split_r.range_begin r.range_begin r.range_end; *) ++ split_range split_r range_size in + +-let new_block s i = + let block_begin = compute_block_begin s i in + let block_end = compute_block_end s i in + let rec b = { +@@ -1365,348 +1340,227 @@ + range_end = block_end; + range_block = b; + range_nuploading = 0; +- range_current_begin = block_begin; + } + in +- + (* lprintf "New block %Ld-%Ld\n" block_begin block_end; *) + split_range range s.s_range_size; +-(* +- let rec iter r = +- lprintf " Range %Ld-%Ld\n" r.range_begin r.range_end; +- match r.range_next with +- None -> () +- | Some r -> iter r +- in +- iter b.block_ranges; +-*) +- s.s_blocks.(i) <- PartialBlock b; if s.s_verified_bitmap.[i] < '1' then - set_bitmap_1 s i; @@ -1220,7 +1641,11 @@ + if debug_all then lprintf_nl "NB[%s]" s.s_verified_bitmap; b -- ++(** Remove an interval from the beginning of a range, adding the size ++ of the removed part to the downloaded amount ++ Closed ranges are removed ++ When last range is removed, mark the block for verification *) + -(*************************************************************************) -(* *) -(* next_range (internal) *) @@ -1248,65 +1673,653 @@ - add_file_downloaded t.t_file (new_downloaded -- old_downloaded) - *) - - (*************************************************************************) - (* *) - (* range_received (internal) *) -@@ -1490,7 +1457,7 @@ - (* lprintf "iter range %Ld-%Ld\n" r.range_begin r.range_end; *) - (try range_received None r chunk_begin chunk_end - with e -> +-(*************************************************************************) +-(* *) +-(* range_received (internal) *) +-(* *) +-(*************************************************************************) +- +-let range_received maybe_t r chunk_begin chunk_end = ++let range_received maybe_t r interval_begin interval_end = + (* lprintf " range_received: %Ld-%Ld for %Ld-%Ld\n" +- chunk_begin chunk_end r.range_begin r.range_end; *) +- if r.range_begin < chunk_end && r.range_end > chunk_begin then begin +- ++ interval_begin interval_end r.range_begin r.range_end; *) ++ (* interval overlap with the beginning of range ? *) ++ (* was: r.range_begin < interval_end && r.range_end > interval_begin *) ++ if r.range_begin >= interval_begin && ++ r.range_begin < interval_end then begin + (* lprintf "... entered\n"; *) +- let new_current_begin = +- max (min chunk_end r.range_end) r.range_current_begin in +- let downloaded = new_current_begin -- r.range_current_begin in +- let b = r.range_block in +- let s = b.block_s in +- add_file_downloaded maybe_t s downloaded; +- b.block_remaining <- b.block_remaining -- downloaded; +- r.range_current_begin <- new_current_begin; +- if r.range_current_begin = r.range_end then begin +- (match r.range_next with +- None -> () +- | Some rr -> rr.range_prev <- r.range_prev); +- (match r.range_prev with +- None -> +- begin +- match r.range_next with +- None -> +- begin +- match s.s_blocks.(b.block_num) with +- PartialBlock _ | EmptyBlock -> +- +- begin +- match s.s_networks with +- [] -> assert false +- | t :: _ when t.t_primary -> +- begin +- match t.t_verifier with +- NoVerification -> +- set_verified_block s b.block_num +- | _ -> +- set_completed_block (Some t) s b.block_num; +- must_verify_block s b.block_num false +- end +- | _ -> () +- end +- | _ -> () +- end +- | Some rr -> b.block_ranges <- rr +- end; +- | Some rr -> rr.range_next <- r.range_next); +- r.range_next <- None; +- r.range_prev <- None; +- end (* else begin +- lprintf " ... new range %Ld-%Ld\n" r.range_current_begin r.range_end; +- end *) +- end +- ++ let new_current_begin = ++ max (min interval_end r.range_end) r.range_begin in ++ let downloaded = new_current_begin -- r.range_begin in ++ let b = r.range_block in ++ let s = b.block_s in ++ add_file_downloaded maybe_t s downloaded; ++ b.block_remaining <- b.block_remaining -- downloaded; ++ r.range_begin <- new_current_begin; ++ if r.range_begin = r.range_end then begin ++ (* range completed, unlink it *) ++ (match r.range_next with ++ | Some rr -> rr.range_prev <- r.range_prev ++ | None -> ()); ++ (match r.range_prev with ++ | Some rr -> rr.range_next <- r.range_next ++ | None -> ++ (* that was the first range of the block *) ++ match r.range_next with ++ | Some rr -> (* fix block's first range *) ++ b.block_ranges <- rr ++ | None -> ++ (* that was the last remaining range of the block *) ++ (match s.s_blocks.(b.block_num) with ++ | PartialBlock _ | EmptyBlock -> ++ (match s.s_networks with ++ | t :: _ -> ++ assert(t.t_primary); ++ (match t.t_verifier with ++ | NoVerification -> ++ set_verified_block s b.block_num ++ | _ -> ++ set_completed_block (Some t) s b.block_num; ++ must_verify_block s b.block_num) ++ | [] -> assert false) ++ | _ -> () )); ++ r.range_next <- None; ++ r.range_prev <- None; ++ end (* else begin ++ lprintf " ... new range %Ld-%Ld\n" r.range_begin r.range_end; ++ end *) ++ end + +-(*************************************************************************) +-(* *) +-(* set_present_block (internal) *) +-(* *) +-(*************************************************************************) ++(** Remove an interval from the ranges of a block, calling ++ range_received over all of them + +-(* Assumption: we never download ranges from the middle, so present chunks +- can only overlap the beginning of a range *) +-let set_present_block b chunk_begin chunk_end = +- let rec iter r = +- let range_next = r.range_next in +-(* lprintf "iter range %Ld-%Ld\n" r.range_begin r.range_end; *) +- (try range_received None r chunk_begin chunk_end +- with e -> - lprintf_nl () "EXCEPTION IN range_received: %s" -+ lprintf_nl "EXCEPTION IN range_received: %s" - (Printexc2.to_string e); - exit 2); - match range_next with -@@ -1512,7 +1479,7 @@ +- (Printexc2.to_string e); +- exit 2); +- match range_next with +- None -> () +- | Some rr -> +- iter rr +- in +-(* lprintf "BEFORE: "; print_block b; *) +- iter b.block_ranges; +-(* lprintf "AFTER: "; print_block b *) +- () ++ Assumption: we never download ranges from the middle, so present ++ intervals can only overlap the beginning of ranges + ++ A (double linked) list is definitely not the most efficient ++ datastructure for this operation... *) + +-(*************************************************************************) +-(* *) +-(* set_present *) +-(* *) +-(*************************************************************************) ++let set_present_block b interval_begin interval_end = ++ iter_block_ranges (fun r -> ++ range_received None r interval_begin interval_end) b - let set_present s chunks = +-let set_present s chunks = ++(** Remove a list of intervals from the ranges of a swarmer *) - apply_intervals s (fun i block_begin block_end chunk_begin chunk_end -> -+ iter_intervals s (fun i block_begin block_end chunk_begin chunk_end -> ++let set_present s intervals = ++ iter_intervals s (fun i block_begin block_end interval_begin interval_end -> (* lprintf "interval: %Ld-%Ld in block %d [%Ld-%Ld]\n" - chunk_begin chunk_end i block_begin block_end; *) - match s.s_blocks.(i) with -@@ -1584,10 +1551,10 @@ +- chunk_begin chunk_end i block_begin block_end; *) +- match s.s_blocks.(i) with +- EmptyBlock -> ++ interval_begin interval_end i block_begin block_end; *) ++ match s.s_blocks.(i) with ++ | EmptyBlock -> + (* lprintf " EmptyBlock"; *) +- if block_begin >= chunk_begin && block_end <= chunk_end then +- begin ++ if block_begin >= interval_begin && block_end <= interval_end ++ then begin + (* lprintf " --> CompleteBlock\n"; *) +- s.s_blocks.(i) <- CompleteBlock; +- must_verify_block s i false; +- add_file_downloaded None s (block_end -- block_begin) +- end +- else ++ s.s_blocks.(i) <- CompleteBlock; ++ must_verify_block s i; ++ add_file_downloaded None s (block_end -- block_begin) ++ end ++ else + let b = new_block s i in + (* lprintf " ... set_present_block\n"; *) +- set_present_block b chunk_begin chunk_end +- | PartialBlock b -> ++ set_present_block b interval_begin interval_end ++ | PartialBlock b -> + (* lprintf " PartialBlock\n"; *) +- set_present_block b chunk_begin chunk_end +- | _ -> ++ set_present_block b interval_begin interval_end ++ | CompleteBlock | VerifiedBlock -> + (* lprintf " Other\n"; *) +- () +- ) chunks +- +-(*************************************************************************) +-(* *) +-(* end_present (internal) *) +-(* *) +-(*************************************************************************) ++ () ++ ) intervals + +-let rec end_present present begin_present end_file list = +- match list with +- [] -> +- let present = +- if begin_present = end_file then present else +- (begin_present, end_file) :: present +- in +- List.rev present +- | (begin_absent, end_absent) :: tail -> +- let present = +- if begin_present = begin_absent then present +- else (begin_present, begin_absent) :: present +- in +- end_present present end_absent end_file tail ++(** reverse absent/present in the list and call set_present *) + +-(*************************************************************************) +-(* *) +-(* set_absent *) +-(* *) +-(*************************************************************************) ++let set_absent s list_absent = ++(** Build the complementary list of intervals of [intervals] in ++ [set_begin, set_end[ *) ++ let rec complementary acc set_begin set_end intervals = ++ match intervals with ++ | [] -> ++ let acc = ++ if set_begin = set_end then acc else ++ (set_begin, set_end) :: acc ++ in ++ List.rev acc ++ | (interval_begin, interval_end) :: other_intervals -> ++ let acc = ++ if set_begin = interval_begin then acc ++ else (set_begin, interval_begin) :: acc ++ in ++ complementary acc interval_end set_end other_intervals in ++ let list_present = complementary [] Int64.zero s.s_size list_absent in ++ set_present s list_present + +-let set_absent s list = +-(* reverse absent/present in the list and call set_present *) +- let list = +- match list with [] -> [ Int64.zero, s.s_size ] +- | (t1,t2) :: tail -> +- if t1 = zero then +- end_present [] t2 s.s_size tail +- else +- end_present [zero, t1] t2 s.s_size tail +- in +- set_present s list ++let intervals_to_string s intervals = ++ match intervals with ++ | AvailableIntervals intervals -> ++ let st = String.make (Array.length s.s_blocks) '0' in ++ iter_intervals s (fun i _ _ _ _ -> st.[i] <- '1') intervals; ++ st ++ | AvailableBitv b -> Bitv.to_string b + + (*************************************************************************) + (* *) +-(* chunks_to_string (internal) *) ++(* Uploaders *) + (* *) + (*************************************************************************) - let chunks_to_string s chunks = - match chunks with +-let chunks_to_string s chunks = +- match chunks with - AvailableRanges chunks -> -+ AvailableIntervals chunks -> - begin - let st = String.make (Array.length s.s_blocks) '0' in +- begin +- let st = String.make (Array.length s.s_blocks) '0' in - apply_intervals s (fun i block_begin block_end chunk_begin chunk_end -> st.[i] <- '1') chunks; -+ iter_intervals s (fun i block_begin block_end chunk_begin chunk_end -> st.[i] <- '1') chunks; - st - end - | AvailableCharBitmap st -> st -@@ -1610,11 +1577,11 @@ - - begin - match chunks with +- st +- end +- | AvailableCharBitmap st -> st +- | AvailableBitv b -> Bitv.to_string b ++(** if not [up_declared], ++ sets [up_intervals], [up_complete_blocks], [up_ncomplete], ++ [up_partial_blocks], [up_npartial] according to [intervals], ++ resets [up_block], [up_block_begin], [up_block_end], and calls ++ [client_has_bitmap] on associated client. + +-(*************************************************************************) +-(* *) +-(* update_uploader_chunks (internal) *) +-(* *) +-(*************************************************************************) ++ My feeling is that if all those fields only make sense when ++ up_declared is true, they should be regrouped in a record option. ++*) + +-let update_uploader_chunks up chunks = +- if not up.up_declared then ++let set_uploader_intervals up intervals = ++ if up.up_declared then ++ lprintf_nl "set_uploader_intervals: called on an already declared uploader\n" ++ else + let t = up.up_t in + let s = t.t_s in +-(* INVARIANT: complete_blocks must be in reverse order *) ++ (* INVARIANT: complete_blocks must be in reverse order *) + +- let complete_blocks = ref [] in +- let partial_blocks = ref [] in ++ let complete_blocks = ref [] in ++ let partial_blocks = ref [] in + +- begin +- match chunks with - AvailableRanges chunks -> -+ AvailableIntervals chunks -> ++ let incr_availability s i = ++ s.s_availability.(i) <- s.s_availability.(i) + 1 in - apply_intervals s (fun i block_begin block_end -+ iter_intervals s (fun i block_begin block_end - chunk_begin chunk_end -> +- chunk_begin chunk_end -> -(* lprintf "apply_intervals %d %Ld-%Ld %Ld-%Ld\n" +- i block_begin block_end chunk_begin chunk_end; *) +- s.s_availability.(i) <- s.s_availability.(i) + 1; ++ (match intervals with ++ | AvailableIntervals intervals -> ++ iter_intervals s (fun i block_begin block_end interval_begin interval_end -> +(* lprintf "iter_intervals %d %Ld-%Ld %Ld-%Ld\n" - i block_begin block_end chunk_begin chunk_end; *) - s.s_availability.(i) <- s.s_availability.(i) + 1; ++ i block_begin block_end interval_begin interval_end; *) ++ incr_availability s i; + +- match s.s_blocks.(i) with +- CompleteBlock | VerifiedBlock -> () +- | _ -> +- if block_begin = chunk_begin && block_end = chunk_end then +- complete_blocks := i :: !complete_blocks +- else +- partial_blocks := +- (i, chunk_begin, chunk_end) :: !partial_blocks +- ) chunks; ++ match s.s_blocks.(i) with ++ | CompleteBlock | VerifiedBlock -> () ++ | EmptyBlock | PartialBlock _ -> ++ if block_begin = interval_begin && block_end = interval_end then ++ complete_blocks := i :: !complete_blocks ++ else ++ partial_blocks := ++ (i, interval_begin, interval_end) :: !partial_blocks ++ ) intervals + +- | AvailableCharBitmap string -> +- for i = 0 to String.length string - 1 do +- if string.[i] = '1' then +- List.iter (fun i -> +- s.s_availability.(i) <- s.s_availability.(i) + 1; +- complete_blocks := i :: !complete_blocks +- ) t.t_blocks_of_chunk.(i) +- done; +- | AvailableBitv bitmap -> +- for i = 0 to Bitv.length bitmap - 1 do +- if Bitv.get bitmap i then +- List.iter (fun i -> +- s.s_availability.(i) <- s.s_availability.(i) + 1; +- complete_blocks := i :: !complete_blocks +- ) t.t_blocks_of_chunk.(i) +- done; +- end; ++ | AvailableBitv bitmap -> ++ Bitv.iteri_true (fun i -> ++ List.iter (fun j -> ++ incr_availability s j; ++ complete_blocks := j :: !complete_blocks ++ ) t.t_blocks_of_chunk.(i) ++ ) bitmap ++ ); + +- List.iter (fun i -> ++ List.iter (fun i -> + (* s.s_last_seen.(i) <- BasicSocket.last_time (); *) ++ let i = t.t_chunk_of_block.(i) in ++ t.t_last_seen.(i) <- BasicSocket.last_time () ++ ) !complete_blocks; + +- let i = t.t_chunk_of_block.(i) in +- t.t_last_seen.(i) <- BasicSocket.last_time () +- +- ) !complete_blocks; +- +- let complete_blocks = Array.of_list !complete_blocks in +- let partial_blocks = Array.of_list !partial_blocks in +- up.up_chunks <- chunks; +- +- up.up_complete_blocks <- complete_blocks; +- up.up_ncomplete <- Array.length complete_blocks; +- +- if Array.length partial_blocks > 0 then +- lprintf_nl () "WARNING: partial_blocks = %d" (Array.length partial_blocks); +- up.up_partial_blocks <- partial_blocks; +- up.up_npartial <- Array.length partial_blocks; +- +- up.up_block <- None; +- up.up_block_begin <- zero; +- up.up_block_end <- zero; +- +- up.up_declared <- true; +- +- let bm = chunks_to_string s chunks in +- client_has_bitmap up.up_client up.up_t.t_file bm; +- +- if debug_all then print_uploader up +- +-(*************************************************************************) +-(* *) +-(* clean_uploader_chunks (internal) *) +-(* *) +-(*************************************************************************) +- ++ let complete_blocks = Array.of_list !complete_blocks in ++ let partial_blocks = Array.of_list !partial_blocks in ++ up.up_intervals <- intervals; + +-let clean_uploader_chunks up = ++ up.up_complete_blocks <- complete_blocks; ++ up.up_ncomplete <- Array.length complete_blocks; ++ ++ if Array.length partial_blocks > 0 then ++ lprintf_nl "WARNING: partial_blocks = %d" (Array.length partial_blocks); ++ up.up_partial_blocks <- partial_blocks; ++ up.up_npartial <- Array.length partial_blocks; + +- if up.up_declared then ++ up.up_block <- None; ++ up.up_block_begin <- zero; ++ up.up_block_end <- zero; + +- let decr_availability s i = +- s.s_availability.(i) <- s.s_availability.(i) - 1 +- in +-(* lprintf "clean_uploader_chunks:\n"; *) ++ up.up_declared <- true; ++ ++ let bm = intervals_to_string s intervals in ++ client_has_bitmap up.up_client up.up_t.t_file bm; + +- let t = up.up_t in +- let s = t.t_s in +- for i = 0 to Array.length up.up_complete_blocks - 1 do +-(* lprintf "decr_availability complete[%d] %d\n" i +- up.up_complete_blocks.(i); *) +- decr_availability s up.up_complete_blocks.(i) +- done; +- for i = 0 to Array.length up.up_partial_blocks - 1 do +- let b,_,_ = up.up_partial_blocks.(i) in +-(* lprintf "decr_availability partial[%d] %d\n" i b; *) +- decr_availability s b +- done; +- clear_uploader_block up; +- up.up_declared <- false ++ if debug_all then print_uploader up -@@ -1662,7 +1629,7 @@ - up.up_ncomplete <- Array.length complete_blocks; + (*************************************************************************) + (* *) +@@ -1714,15 +1568,14 @@ + (* *) + (*************************************************************************) - if Array.length partial_blocks > 0 then -- lprintf_nl () "WARNING: partial_blocks = %d" (Array.length partial_blocks); -+ lprintf_nl "WARNING: partial_blocks = %d" (Array.length partial_blocks); - up.up_partial_blocks <- partial_blocks; - up.up_npartial <- Array.length partial_blocks; +-let register_uploader t client chunks = +- ++let register_uploader t client intervals = + let up = + { + up_t = t; + up_client = client; + + up_declared = false; +- up_chunks = chunks; ++ up_intervals = intervals; + + up_complete_blocks = [||]; + up_ncomplete = 0; +@@ -1733,11 +1586,12 @@ + up_block = None; + up_block_begin = zero; + up_block_end = zero; ++ + up_ranges = []; + } + in + HU.add uploaders_by_num up; +- update_uploader_chunks up chunks; ++ set_uploader_intervals up intervals; + up + + (*************************************************************************) +@@ -1746,34 +1600,63 @@ + (* *) + (*************************************************************************) + +-let unregister_uploader up = +- clean_uploader_chunks up; +- clear_uploader_block up; +- clear_uploader_ranges up ++let clear_uploader_ranges up = ++ List.iter (fun (_,_,r) -> ++ if r.range_nuploading > 0 then ++ r.range_nuploading <- r.range_nuploading - 1 ++ else ++ lprintf_nl "clear_uploader_ranges: some range_nuploading was about to become negative\n" ++ ) up.up_ranges; ++ up.up_ranges <- [] + +-(*************************************************************************) +-(* *) +-(* update_uploader *) +-(* *) +-(*************************************************************************) ++let clear_uploader_block up = ++ match up.up_block with ++ | None -> () ++ | Some b -> ++ let num = b.block_num in ++ let t = up.up_t in ++ let s = t.t_s in ++ if s.s_nuploading.(num) > 0 then ++ s.s_nuploading.(num) <- s.s_nuploading.(num) - 1 ++ else ++ lprintf_nl "clear_uploader_block: some s_nuploading was about to become negative\n"; ++ up.up_block <- None; ++ up.up_block_begin <- zero; ++ up.up_block_end <- zero + +-let update_uploader up chunks = ++let clear_uploader_intervals up = ++ if up.up_declared then ++ let decr_availability s i = ++ if s.s_availability.(i) > 0 then ++ s.s_availability.(i) <- s.s_availability.(i) - 1 ++ else ++ lprintf_nl "clear_uploader_intervals: some s_availability was about to become negative\n" in ++(* lprintf "clean_uploader_chunks:\n"; *) ++ let t = up.up_t in ++ let s = t.t_s in ++ Array.iter (decr_availability s) up.up_complete_blocks; ++ up.up_complete_blocks <- [||]; ++ up.up_ncomplete <- 0; ++ Array.iter (fun (b,_,_) -> decr_availability s b) up.up_partial_blocks; ++ up.up_partial_blocks <- [||]; ++ up.up_npartial <- 0; ++ clear_uploader_block up; ++ up.up_declared <- false + +- clean_uploader_chunks up; +- update_uploader_chunks up chunks ++let update_uploader_intervals up intervals = ++ clear_uploader_intervals up; ++ set_uploader_intervals up intervals + +-(*************************************************************************) +-(* *) +-(* print_uploaders *) +-(* *) +-(*************************************************************************) ++let unregister_uploader up = ++ clear_uploader_intervals up; ++ clear_uploader_ranges up + +-let print_uploaders s = +- let nblocks = Array.length s.s_blocks in +- for i = 0 to nblocks - 1 do ++(** (debug) output the uploaders of a swarmer to current log *) + +- match s.s_blocks.(i) with +- EmptyBlock -> lprintf "_" ++let print_uploaders s = ++ Array.iteri (fun i b -> ++ match b with ++ | EmptyBlock -> lprintf "_" + | CompleteBlock -> lprintf "C" + | VerifiedBlock -> lprintf "V" + | PartialBlock b -> +@@ -1781,127 +1664,128 @@ + lprintf "X" + else + lprintf "%d" s.s_nuploading.(i) +- done; ++ ) s.s_blocks; + lprint_newline (); +- for i = 0 to nblocks - 1 do +- +- match s.s_blocks.(i) with +- EmptyBlock -> lprintf "_" ++ Array.iteri (fun i b -> ++ match b with ++ | EmptyBlock -> lprintf "_" + | CompleteBlock -> lprintf "C" + | VerifiedBlock -> lprintf "V" + | PartialBlock b -> + lprintf "{ %d : %d=" b.block_num + s.s_nuploading.(b.block_num); +- +- let rec iter_range r = +- lprintf "(%d)" r.range_nuploading; +- match r.range_next with +- None -> () +- | Some rr -> iter_range rr +- in +- iter_range b.block_ranges; ++ iter_block_ranges (fun r -> ++ lprintf "(%d)" r.range_nuploading ++ ) b; + lprintf " }"; +- +- done; ++ ) s.s_blocks; + lprint_newline () + +-(*************************************************************************) +-(* *) +-(* permute_and_return (internal) *) +-(* *) +-(*************************************************************************) ++(** swap elements n and up_ncomplete-1 in up_complete_blocks, then ++ decrease up_ncomplete block, return up_ncomplete block, converting ++ it to a PartialBlock as needed. ++ global effect is that up_complete_blocks array virtually contains ++ two disctinct lists, with elements flowing from one to the other, ++ without any allocation needed. ++ ++ 0 .. up_ncomplete-1 : not yet returned by permute_and_return ++ up_ncomplete .. Array.length up_complete_blocks - 1 : already returned ++*) -@@ -1826,10 +1793,10 @@ + let permute_and_return up n = ++ assert (n <= up.up_ncomplete-1); + let b = up.up_complete_blocks.(n) in + if debug_all then lprintf "permute_and_return %d <> %d" n b; +- up.up_complete_blocks.(n) <- up.up_complete_blocks.(up.up_ncomplete-1); +- up.up_complete_blocks.(up.up_ncomplete-1) <- b; ++ if n < up.up_ncomplete then begin ++ up.up_complete_blocks.(n) <- up.up_complete_blocks.(up.up_ncomplete-1); ++ up.up_complete_blocks.(up.up_ncomplete-1) <- b ++ end; + up.up_ncomplete <- up.up_ncomplete - 1; + let t = up.up_t in + let s = t.t_s in + match s.s_blocks.(b) with +- EmptyBlock -> ++ | EmptyBlock -> + let b = new_block s b in + b, b.block_begin, b.block_end | PartialBlock b -> b, b.block_begin, b.block_end | VerifiedBlock -> @@ -1318,79 +2331,805 @@ + lprintf_nl "ERROR: complete block in permute_and_return %d\n" b; assert false +-(*************************************************************************) +-(* *) +-(* LinearStrategy.select_block (internal) *) +-(* *) +-(*************************************************************************) ++(** find a block in up_complete_blocks that's not already ++ CompleteBlock or VerifiedBlock. ++ If none can be found, do the same with up_partial_blocks. ++ If none can be found still, raise Not_found exception ++ ++ up_ncomplete and up_npartial are used as in the same way as in ++ permute_and_return, but no element is ever permuted. + +-module LinearStrategy = struct +- let select_block up = +- let rec iter_complete up = +- let n = up.up_ncomplete in +- if n = 0 then iter_partial up +- else +- let b = up.up_complete_blocks.(n-1) in +- up.up_ncomplete <- n-1; +- let t = up.up_t in +- let s = t.t_s in +- match s.s_blocks.(b) with +- CompleteBlock | VerifiedBlock -> +- iter_complete up +- | PartialBlock b -> +- b, b.block_begin, b.block_end +- | EmptyBlock -> +- let b = new_block s b in +- b, b.block_begin, b.block_end ++ Since set_uploader_intervals puts the blocks with the lowest ++ offsets at the end of up_complete_blocks and up_partial_blocks, ++ this also selects the blocks in increasing offsets order. ++*) + +- and iter_partial up = +- let n = up.up_npartial in +- if n = 0 then raise Not_found; +- let b, block_begin, block_end = up.up_partial_blocks.(n-1) in +- let t = up.up_t in +- let s = t.t_s in +- match s.s_blocks.(b) with +- CompleteBlock | VerifiedBlock -> +- iter_partial up +- | PartialBlock b -> +- b, block_begin, block_end +- | EmptyBlock -> +- let b = new_block s b in +- b, block_begin, block_end +- in +- iter_complete up +- end ++let linear_select_block up = ++ let rec iter_partial up = ++ let n = up.up_npartial in ++ if n = 0 then raise Not_found; ++ let b, block_begin, block_end = up.up_partial_blocks.(n-1) in ++ let t = up.up_t in ++ let s = t.t_s in ++ (* priority bitmap <> 0 here ? *) ++ match s.s_blocks.(b) with ++ | CompleteBlock | VerifiedBlock -> ++ iter_partial up ++ | PartialBlock b -> ++ b, block_begin, block_end ++ | EmptyBlock -> ++ let b = new_block s b in ++ b, block_begin, block_end in ++ let rec iter_complete up = ++ let n = up.up_ncomplete in ++ if n = 0 then iter_partial up ++ else ++ let b = up.up_complete_blocks.(n-1) in ++ up.up_ncomplete <- n-1; ++ let t = up.up_t in ++ let s = t.t_s in ++ (* priority bitmap <> 0 here ? *) ++ match s.s_blocks.(b) with ++ | CompleteBlock | VerifiedBlock -> ++ iter_complete up ++ | PartialBlock b -> ++ b, b.block_begin, b.block_end ++ | EmptyBlock -> ++ let b = new_block s b in ++ b, b.block_begin, b.block_end ++ in ++ iter_complete up + +-(*************************************************************************) +-(* *) +-(* should_download_block (internal) *) +-(* *) +-(*************************************************************************) ++(** Check whether block [n] of swarmer [s] is not completed yet, ++ calling chunk verification first if block still need verification *) + + let should_download_block s n = + (* lprintf "should_download_block %d\n" n; *) + let result = + match s.s_verified_bitmap.[n] with +- '2' -> +- begin +- match s.s_networks with +- [] -> assert false +- | t :: _ when t.t_primary -> +- begin +- try +- let n = t.t_chunk_of_block.(n) in +- if t.t_converted_verified_bitmap.[n] = '2' then +- verify_chunk t n +- with VerifierNotReady -> () +- end +- | _ -> () +- end; +- s.s_verified_bitmap.[n] < '2' + | '0' | '1' -> true +- | _ -> false ++ | '2' -> ++ (match s.s_networks with ++ | t :: _ -> ++ assert(t.t_primary); ++ (try ++ let n = t.t_chunk_of_block.(n) in ++ if t.t_converted_verified_bitmap.[n] = '2' then ++ verify_chunk t n ++ with VerifierNotReady -> ()); ++ | [] -> assert false); ++ s.s_verified_bitmap.[n] < '2' ++ | '3' -> false ++ | _ -> assert false + in + (* if result then + lprintf "should_download_block %d\n" n; *) +@@ -1913,264 +1797,204 @@ + (* *) (*************************************************************************) -@@ -1917,7 +1884,7 @@ - let random_int max = - let x = Random.int max in +-exception BlockFound of int ++(* Would it be faster not to build those records, and use functions of ++ the block number ? *) + +-let random_int max = +- let x = Random.int max in - if debug_all then lprintf_nl () "(Random %d -> %d)" max x; -+ if debug_all then lprintf_nl "(Random %d -> %d)" max x; - x +- x ++type choice = { ++ choice_num : int; ++ choice_user_priority : int; ++ choice_nuploaders : int; ++ choice_size : int64; ++ choice_remaining : int64; ++ choice_remaining_per_uploader : int64; ++ choice_other_complete : int Lazy.t; (* ...blocks in the same chunk *) ++ choice_availability : int; ++} ++ ++(* based on Array.fold_left code *) ++let array_fold_lefti f x a = ++ let r = ref x in ++ for i = 0 to Array.length a - 1 do ++ r := f !r i (Array.unsafe_get a i) ++ done; ++ !r let select_block up = -@@ -1955,7 +1922,7 @@ - (************* Try to download the movie index and the first minute to - allow preview of the file as soon as possible *) - + let t = up.up_t in + let s = t.t_s in + try + match s.s_strategy with +- LinearStrategy -> +- LinearStrategy.select_block up ++ | LinearStrategy -> ++ linear_select_block up + | _ -> + if up.up_ncomplete = 0 && up.up_npartial = 0 then raise Not_found; + +-(************** +- +-This strategy sucks. It has to be improved. +-Important: +-1) never give a block to 2 clients if another one has 0 client. +-2) try to complete partial blocks as soon as possible. +-3) comfigure the chooser depending on the network (maybe BT might +-better work at the beginning if the first incomplete blocks are offered +- to several clients. +- +-***************) +- +- +- +- if up.up_ncomplete > 1 then begin +-(* let debug_all = true in *) +- try +- +- let rec iter_max_uploaders max_nuploaders = +- let t = up.up_t in +- let nblocks = Array.length s.s_blocks in +- +-(************* Try to download the movie index and the first minute to +- allow preview of the file as soon as possible *) +- - if debug_all then lprintf_nl () "{First}"; -+ if debug_all then lprintf_nl "{First}"; - - let download_first n b = - (* lprintf "download_first %d\n" n; *) -@@ -1989,7 +1956,7 @@ - (************* If the file can be verified, and we don't have a lot of blocks - yet, try to download the partial ones as soon as possible *) - +- +- let download_first n b = +-(* lprintf "download_first %d\n" n; *) +- if +- up.up_complete_blocks.(n) = b && +- s.s_nuploading.(b) < max_nuploaders && +- should_download_block s b then +- raise (BlockFound n) +- in +- +-(* lprintf "up_complete_blocks: %d\n" +- (Array.length up.up_complete_blocks); *) +- +-(* This must be the position of the last block of the file *) +- download_first 0 (nblocks-1); +- +-(* This can be the position of the first block of the file *) +- download_first (up.up_ncomplete-1) 0; +- +-(* This can be the position of the first block of the file *) +- download_first 0 0; +- +-(* This must be the position of the second last block of the file *) +- download_first 0 (nblocks-2); +- +-(* These can be the positions of the second block of the file *) +- download_first 0 1; +- download_first (up.up_ncomplete-1) 1; +- download_first (up.up_ncomplete-2) 1; +- +-(************* If the file can be verified, and we don't have a lot of blocks +- yet, try to download the partial ones as soon as possible *) +- - if debug_all then lprintf_nl () "{PartialBlock}"; -+ if debug_all then lprintf_nl "{PartialBlock}"; +- +- let download_partial max_uploaders = +- let partial_block = ref (-1) in +- let partial_remaining = ref zero in +- for i = 0 to up.up_ncomplete - 1 do +- let n = up.up_complete_blocks.(i) in +- match s.s_blocks.(n) with +- PartialBlock b -> +- if (!partial_block = -1 || +- !partial_remaining > b.block_remaining) && +- s.s_nuploading.(n) < max_uploaders +- then +- begin +- partial_block := i; +- partial_remaining := b.block_remaining +- end +- | _ -> () +- done; +- if !partial_block <> -1 then +- raise (BlockFound !partial_block) +- in +- +- if t.t_verifier <> NoVerification && +- t.t_nverified_blocks < 2 then begin +- download_partial max_nuploaders; +- end; +- +-(************* Download partial chunks from the verification point of view *) +- +- if List.length s.s_networks > 1 then begin +- if debug_all then lprintf_n () "{PartialChunk}"; +- +- let my_t = if t.t_verifier <> NoVerification then t +- else match s.s_networks with t :: _ -> t | _ -> t in +- +- let download_partial max_uploaders = +- let partial_block = ref (-1) in +- let partial_remaining = ref 0 in +- for i = 0 to up.up_ncomplete - 1 do +- let n = up.up_complete_blocks.(i) in +-(* TODO move this after the first if... *) +- let t_index = t.t_chunk_of_block.(n) in +- let bs = List.filter (fun s_index -> +- s.s_verified_bitmap.[s_index] = '2' +- ) t.t_blocks_of_chunk.(t_index) in +- let nbs = List.length bs in +- +-(* TODO remove this *) +- let b = should_download_block s n in ++(* to evaluate the relative rarity of a block, we must compare it to ++ the availability of *all* blocks, not only those available from ++ that uploader *) ++ let sum_availability = Array.fold_left (+) 0 s.s_availability in ++ let mean_availability = sum_availability / Array.length s.s_blocks in + +- if !verbose_swarming then +- lprintf_nl2 " test %d %c %d %b %d" +- n s.s_verified_bitmap.[n] s.s_nuploading.(n) +- b nbs; ++ let my_t = if t.t_verifier <> NoVerification then t else ++ match s.s_networks with ++ | tprim :: _ -> ++ assert(tprim.t_primary); ++ tprim ++ | [] -> assert false in ++ let verification_available = my_t.t_verifier <> NoVerification in + +- if s.s_verified_bitmap.[n] < '2' && +- s.s_nuploading.(n) < max_uploaders && +- should_download_block s n then ++ let several_frontends = List.length s.s_networks > 1 in ++ (* many results may not be useful, evaluate them as needed *) ++ let completed_blocks_in_chunk = ++ Array.init my_t.t_nchunks (fun i -> ++ lazy ( ++ List.fold_left (fun acc b -> ++ if s.s_verified_bitmap.[b] = '2' then acc + 1 else acc ++ ) 0 my_t.t_blocks_of_chunk.(i))) in + +- if (!partial_block = -1 || !partial_remaining < nbs) +- then +- begin +- partial_block := i; +- partial_remaining := nbs +- end +- done; +- if !partial_block <> -1 then begin +- if !verbose_swarming then +- lprintf_n () "PartialChunk won for %d waiting blocks" +- !partial_remaining; +- raise (BlockFound !partial_block) +- end +- in ++ let preview_beginning = 10000000L in ++ let preview_end = (s.s_size ** 98L) // 100L in + +- if my_t.t_verifier <> NoVerification then begin +- download_partial max_nuploaders; +- end; +- end; ++ (* sources_per_chunk was initially for edonkey only *) ++ let data_per_source = 9728000L // (Int64.of_int !!sources_per_chunk) in + +-(************* Download rarest first only if other blocks are much more +- available *) ++ let need_to_complete_some_blocks_quickly = true ++ (* verification_available && t.t_nverified_chunks < 2 *) in + +- if debug_all then lprintf "{Rarest}"; ++ (** > 0 == c1 is best, < 0 = c2 is best, 0 == they're equivalent *) ++ let compare_choices c1 c2 = + +- let sum_availability = ref 0 in +- let min_availability = ref max_int in +- for i = 0 to up.up_ncomplete - 1 do +- let n = up.up_complete_blocks.(i) in +- sum_availability := !sum_availability + +- s.s_availability.(n); +- min_availability := min !min_availability +- s.s_availability.(n); +- done; ++ (* avoid overly unbalanced situations *) ++ let cmp = ++ if c1.choice_remaining_per_uploader < data_per_source || ++ c2.choice_remaining_per_uploader < data_per_source then ++ compare c1.choice_remaining_per_uploader ++ c2.choice_remaining_per_uploader else 0 in ++ if cmp <> 0 then cmp else + +- let mean_availability = +- !sum_availability / up.up_ncomplete in ++ (* Do what Master asked for *) ++ let cmp = compare c1.choice_user_priority c2.choice_user_priority in ++ if cmp <> 0 then cmp else + +- if mean_availability > 5 && !min_availability < 3 then +- for i = 0 to up.up_ncomplete - 1 do +- let n = up.up_complete_blocks.(i) in +- if s.s_availability.(n) < 3 +- && should_download_block s n +- then +- raise (BlockFound i) +- done; ++ (* Pick really rare gems *) ++ let cmp = ++ if mean_availability > 5 && ++ (c1.choice_availability <= 3 || c2.choice_availability <= 3) then ++ compare c2.choice_availability c1.choice_availability ++ else 0 in ++ if cmp <> 0 then cmp else + +-(************* Otherwise, download in random order *) ++ (* try to quickly complete blocks *) ++ let cmp = ++ if need_to_complete_some_blocks_quickly then ++ compare c2.choice_remaining c1.choice_remaining else 0 in ++ if cmp <> 0 then cmp else + +- if debug_all then lprintf "{Random}"; +- let find_random max_uploaders = +- let list = ref [] in +- if debug_all then lprintf " {NC: %d}" up.up_ncomplete; +- for i = 0 to up.up_ncomplete - 1 do +- let n = up.up_complete_blocks.(i) in +- if s.s_nuploading.(n) < max_uploaders +- && should_download_block s n +- then +- list := i :: !list +- done; +- match !list with +- [] -> () +- | [i] -> raise (BlockFound i) +- | list -> +- let array = Array.of_list list in +- raise (BlockFound (array.( +- random_int (Array.length array)))) +- in ++ (* try to quickly complete (and validate) chunks *) ++ let cmp = ++ if verification_available && several_frontends then ++ compare (Lazy.force c1.choice_other_complete) ++ (Lazy.force c2.choice_other_complete) ++ else 0 in ++ if cmp <> 0 then cmp else + +- find_random max_nuploaders ++ (* Can't tell *) ++ 0 in + +-(************* Fall back on linear download if nothing worked *) ++ let best_choices = array_fold_lefti (fun acc n b -> ++ (* priority bitmap <> 0 here ? *) ++ if not (should_download_block s b) then acc else ++ let nchunk = my_t.t_chunk_of_block.(b) in ++ let block_begin = compute_block_begin s b in ++ let block_end = compute_block_end s b in ++ let size = block_end -- block_begin in ++ let remaining = match s.s_blocks.(b) with ++ | EmptyBlock -> size ++ | PartialBlock b -> b.block_remaining ++ | CompleteBlock | VerifiedBlock -> 0L in ++ let nuploaders = s.s_nuploading.(b) in ++ let this_choice = { ++ choice_num = n; ++ choice_user_priority = (* priority bitmap here instead ? *) ++ if block_begin < preview_beginning then 3 else ++ if block_end > preview_end then 2 else 1; ++ choice_nuploaders = nuploaders; ++ choice_size = size; ++ choice_remaining = remaining; ++ choice_remaining_per_uploader = remaining // ++ (Int64.of_int (nuploaders + 1)); (* planned value *) ++ choice_other_complete = completed_blocks_in_chunk.(nchunk); ++ choice_availability = s.s_availability.(b); ++ } in ++ match acc with ++ | [] -> [this_choice] ++ | h :: _ -> ++ (* all the choices in the accumulator are supposed to ++ be equivalent, compare against the first *) ++ let cmp = compare_choices this_choice h in ++ if cmp > 0 then [this_choice] ++ else if cmp < 0 then acc ++ else this_choice :: acc ++ ) [] up.up_complete_blocks in ++ (* what about up_partial_blocks ? ++ currently they're taken care of by linear_select_block ++ fallback below *) + +- in +- iter_max_uploaders !!sources_per_chunk; +- iter_max_uploaders max_int; +- raise Not_found +- with +- BlockFound n -> +- if debug_all then lprintf "\nBlockFound %d\n" +- up.up_complete_blocks.(n); +- permute_and_return up n +- end else +- LinearStrategy.select_block up ++ try ++ let result = ++ match best_choices with ++ | [] -> raise Not_found ++ | [choice] -> choice ++ | _::_ -> ++ let nchoices = List.length best_choices in ++ List.nth best_choices (Random.int nchoices) in ++ let n = result.choice_num in ++ ++ if debug_all then lprintf "\nBlockFound %d\n" ++ up.up_complete_blocks.(n); ++ permute_and_return up n ++ with Not_found -> ++ linear_select_block up + with Not_found -> - let download_partial max_uploaders = - let partial_block = ref (-1) in -@@ -2013,14 +1980,14 @@ - in + (* print_s "NO BLOCK FOUND" s; *) +- raise Not_found ++ raise Not_found - if t.t_verifier <> NoVerification && -- t.t_nverified_blocks < 2 then begin -+ t.t_nverified_chunks < 2 then begin - download_partial max_nuploaders; - end; +-(*************************************************************************) +-(* *) +-(* find_block *) +-(* *) +-(*************************************************************************) ++(** If uploader is associated to a file being downloaded, ++ clear previously selected block (in any) and select best available ++ block, according to block selection strategy ++ @param up the uploader *) - (************* Download partial chunks from the verification point of view *) + let find_block up = + try + if debug_all then begin +- lprintf "C: "; +- for i = 0 to up.up_ncomplete - 1 do +- lprintf "%d " up.up_complete_blocks.(i) +- done; +- end; ++ lprintf "C: "; ++ for i = 0 to up.up_ncomplete - 1 do ++ lprintf "%d " up.up_complete_blocks.(i) ++ done; ++ end; - if List.length s.s_networks > 1 then begin -- if debug_all then lprintf_n () "{PartialChunk}"; -+ if debug_all then lprintf_n "{PartialChunk}"; - - let my_t = if t.t_verifier <> NoVerification then t - else match s.s_networks with t :: _ -> t | _ -> t in -@@ -2058,7 +2025,7 @@ - done; - if !partial_block <> -1 then begin - if !verbose_swarming then -- lprintf_n () "PartialChunk won for %d waiting blocks" -+ lprintf_n "PartialChunk won for %d waiting blocks" - !partial_remaining; - raise (BlockFound !partial_block) - end -@@ -2179,7 +2146,7 @@ + let t = up.up_t in + let s = t.t_s in + match file_state t.t_file with + | FilePaused + | FileAborted _ +- | FileCancelled -> raise Not_found +- | _ -> +- +- ++ | FileCancelled ++ | FileShared ++ | FileNew ++ | FileDownloaded -> ++ raise Not_found ++ | FileDownloading ++ | FileQueued -> + (match up.up_block with +- None -> () +- | Some b -> +- let num = b.block_num in +- s.s_nuploading.(num) <- s.s_nuploading.(num) - 1; +- up.up_block <- None; ++ | None -> () ++ | Some b -> ++ let num = b.block_num in ++ s.s_nuploading.(num) <- s.s_nuploading.(num) - 1; ++ up.up_block <- None; + ); + +- let (b,block_begin,block_end) (* as result *) = select_block up in ++ let b, block_begin, block_end = select_block up in + let num = b.block_num in + s.s_nuploading.(num) <- s.s_nuploading.(num) + 1; + up.up_block <- Some b; +@@ -2179,60 +2003,33 @@ if debug_all then lprintf " = %d \n" num; b with e -> - if debug_all then lprintf_nl () "Exception %s" (Printexc2.to_string e); -+ if debug_all then lprintf_nl "Exception %s" (Printexc2.to_string e); - raise e +- raise e +- +-(*************************************************************************) +-(* *) +-(* clean_ranges (internal) *) +-(* *) +-(*************************************************************************) ++ if debug_all then lprintf_nl "Exception %s" (Printexc2.to_string e); ++ raise e + +-let clean_ranges up = ++(** Remove completed ranges from an uploader's range list, and ++ decrease their reference counter *) + +- let rec iter list left = +- match list with +- [] -> List.rev left +- | ((_,_,r) as rr) :: tail -> +- iter tail +- (if r.range_current_begin < r.range_end then rr :: left +- else begin +- r.range_nuploading <- r.range_nuploading - 1; +- left +- end) +- in +- up.up_ranges <- iter up.up_ranges [] ++let remove_completed_uploader_ranges up = ++ let not_completed_ranges, completed_ranges = ++ List.partition (fun (_,_,r) -> ++ r.range_begin < r.range_end) up.up_ranges in ++ up.up_ranges <- not_completed_ranges; ++ List.iter (fun (_,_,r) -> ++ r.range_nuploading <- r.range_nuploading - 1) completed_ranges + +-(*************************************************************************) +-(* *) +-(* current_ranges *) +-(* *) +-(*************************************************************************) ++(** uploader accessors *) + + let current_ranges up = up.up_ranges + +-(*************************************************************************) +-(* *) +-(* current_block *) +-(* *) +-(*************************************************************************) +- + let current_block up = + match up.up_block with +- None -> raise Not_found ++ | None -> raise Not_found + | Some b -> b +-(*************************************************************************) +-(* *) +-(* in_uploader_ranges *) +-(* *) +-(*************************************************************************) ++(** Check whether a range is in a list *) + +-let rec in_uploader_ranges r list = +- match list with +- [] -> false +- | (_,_,r') :: tail when r' == r -> true +- | _ :: tail -> in_uploader_ranges r tail ++let in_uploader_ranges r list = ++ List.exists (fun (_,_,r') -> r' == r) list + + (*************************************************************************) + (* *) +@@ -2240,262 +2037,200 @@ + (* *) (*************************************************************************) -@@ -2274,7 +2241,7 @@ - up.up_ranges <- up.up_ranges @ [key]; - r.range_nuploading <- r.range_nuploading + 1; - if r.range_current_begin = r.range_end then + ++let uploader_ranges_fold_left f acc l = ++ let rec aux acc l = ++ match l with ++ | [] -> acc ++ | h :: q -> aux (f acc h) q ++ in aux acc l ++ + let find_range up = +- clean_ranges up; ++ remove_completed_uploader_ranges up; + + let b = + match up.up_block with +- None -> raise Not_found ++ | None -> raise Not_found + | Some b -> b + in +- let r = b.block_ranges in +- + let t = up.up_t in +- + match file_state t.t_file with + | FilePaused + | FileAborted _ +- | FileCancelled -> raise Not_found +- | _ -> +- +- let rec iter limit r = +- +-(* let use a very stupid heuristics: ask for the first non-used range. +-we thus might put a lot of clients on the same range ! +-*) ++ | FileCancelled ++ | FileShared ++ | FileNew ++ | FileDownloaded -> ++ raise Not_found ++ | FileDownloading ++ | FileQueued -> ++ (* pick the first correct range with fewest uploaders *) ++ let best_range = ++ let rec iter acc r = ++ let better_found = ++ if in_uploader_ranges r up.up_ranges || ++ r.range_begin = r.range_end || ++ r.range_begin < up.up_block_begin || ++ r.range_end > up.up_block_end then ++ false ++ else ++ match acc with ++ | None -> true ++ | Some best_range -> ++ best_range.range_nuploading > r.range_nuploading in ++ (* fast exit, and why I didn't use an iterator :/ *) ++ if better_found && r.range_nuploading = 0 then Some r ++ else ++ let acc = if better_found then Some r else acc in ++ match r.range_next with ++ | None -> acc ++ | Some rr -> iter acc rr in ++ iter None b.block_ranges in ++ match best_range with ++ | None -> raise Not_found ++ | Some r -> ++ let key = r.range_begin, r.range_end, r in ++ up.up_ranges <- up.up_ranges @ [key]; ++ r.range_nuploading <- r.range_nuploading + 1; ++ key + +- if not (in_uploader_ranges r up.up_ranges) && +- r.range_current_begin < r.range_end && +- r.range_current_begin >= up.up_block_begin && +- r.range_end <= up.up_block_end && +- r.range_nuploading < limit +- then begin +- let key = r.range_current_begin, r.range_end, r in +- up.up_ranges <- up.up_ranges @ [key]; +- r.range_nuploading <- r.range_nuploading + 1; +- if r.range_current_begin = r.range_end then - lprintf_nl () "error: range is empty error"; -+ lprintf_nl "error: range is empty error"; - key - end else - match r.range_next with -@@ -2309,7 +2276,7 @@ +- key +- end else +- match r.range_next with +- None -> raise Not_found +- | Some rr -> iter limit rr +- in +- try +-(* try normal ranges *) +- iter !!sources_per_chunk r +- with Not_found -> +-(* force maximal uploading otherwise to finish it *) +- iter max_int r ++(** range accessor(s) *) + +-(*************************************************************************) +-(* *) +-(* range_range *) +-(* *) +-(*************************************************************************) ++let range_range r = (r.range_begin, r.range_end) + +-let range_range r = (r.range_current_begin, r.range_end) ++(** Data has been received from uploader [up]. Transfer data to file ++ and update uploader ranges. ++ Data = String.sub [str] [string_begin] [string_len] *) + +-(*************************************************************************) +-(* *) +-(* received *) +-(* *) +-(*************************************************************************) ++let received up file_begin str string_begin string_len = ++ assert (string_begin >= 0); ++ assert (string_len >= 0); ++ assert (string_begin + string_len <= String.length str); + +-let received (up : uploader) (file_begin : Int64.t) +- (str:string) (string_begin:int) (string_len:int) = ++(* ++ let debug_bad_write r string_pos = ++ if !verbose then begin ++ let t = up.up_t in ++ let s = t.t_s in ++ lprintf_nl "BAD WRITE in %s for range %Ld-%Ld (string_pos %d)" ++ (file_best_name t.t_file) r.range_begin r.range_end string_pos; ++ lprintf_nl " received: file_pos:%Ld string:%d %d" ++ file_begin string_begin string_len; ++ lprintf_nl " ranges:"; ++ List.iter (fun (_,_,r) -> ++ lprintf_n " range: %Ld-%Ld" ++ r.range_begin ++ r.range_end; ++ (match r.range_next with ++ | None -> () ++ | Some rr -> ++ lprintf " next: %Ld" rr.range_begin); ++ (match r.range_prev with ++ | None -> () ++ | Some rr -> ++ lprintf " prev: %Ld" rr.range_begin); ++ lprint_newline (); ++ let b = r.range_block in ++ lprintf_n " block: %d[%c] %Ld-%Ld [%s]" ++ b.block_num ++ s.s_verified_bitmap.[b.block_num] ++ b.block_begin b.block_end ++ (match s.s_blocks.(b.block_num) with ++ | EmptyBlock -> "empty" ++ | PartialBlock _ -> "partial" ++ | CompleteBlock -> "complete" ++ | VerifiedBlock -> "verified" ++ ); ++ let br = b.block_ranges in ++ lprintf " first range: %Ld(%Ld)" ++ br.range_begin ++ (br.range_end -- br.range_begin); ++ lprint_newline (); ++ ) up.up_ranges ++ end; ++ if !exit_on_error then exit 2 in *) + + if string_len > 0 then let file_end = file_begin ++ (Int64.of_int string_len) in if !verbose_swarming then @@ -1399,70 +3138,262 @@ (* TODO: check that everything we received has been required *) let t = up.up_t in -@@ -2340,13 +2307,13 @@ - string_len < string_length then begin - if !verbose then - begin + let s = t.t_s in +- try +- +- List.iter (fun (_,_,r) -> +- if r.range_current_begin < file_end && +- r.range_end > file_begin then begin +- +- let file_end = min file_end r.range_end in +- let written_len = file_end -- r.range_current_begin in +- +- begin +- match file_state t.t_file with +- | FilePaused +- | FileAborted _ +- | FileCancelled -> () +- | _ -> +- +- let string_pos = string_begin + +- Int64.to_int (r.range_current_begin -- file_begin) in +- let string_length = Int64.to_int written_len in +- +- if +- string_pos < 0 || +- string_pos < string_begin || +- string_len < string_length then begin +- if !verbose then +- begin - lprintf_nl () "BAD WRITE in %s for range %Ld-%Ld (string_pos %d)" -+ lprintf_nl "BAD WRITE in %s for range %Ld-%Ld (string_pos %d)" - (file_best_name t.t_file) r.range_begin r.range_end string_pos; +- (file_best_name t.t_file) r.range_begin r.range_end string_pos; - lprintf_nl () " received: file_pos:%Ld string:%d %d" -+ lprintf_nl " received: file_pos:%Ld string:%d %d" - file_begin string_begin string_len; +- file_begin string_begin string_len; - lprintf_nl () " ranges:"; -+ lprintf_nl " ranges:"; - List.iter (fun (_,_,r) -> +- List.iter (fun (_,_,r) -> - lprintf_n () " range: %Ld-[%Ld]-%Ld" -+ lprintf_n " range: %Ld-[%Ld]-%Ld" - r.range_begin r.range_current_begin - r.range_end; - (match r.range_next with -@@ -2359,7 +2326,7 @@ - lprintf " prev: %Ld" rr.range_begin); - lprint_newline (); - let b = r.range_block in +- r.range_begin r.range_current_begin +- r.range_end; +- (match r.range_next with +- None -> () +- | Some rr -> +- lprintf " next: %Ld" rr.range_begin); +- (match r.range_prev with +- None -> () +- | Some rr -> +- lprintf " prev: %Ld" rr.range_begin); +- lprint_newline (); +- let b = r.range_block in - lprintf_n () " block: %d[%c] %Ld-%Ld [%s]" -+ lprintf_n " block: %d[%c] %Ld-%Ld [%s]" - b.block_num - s.s_verified_bitmap.[b.block_num] - b.block_begin b.block_end -@@ -2408,7 +2375,7 @@ - - let rec iter_block_out i block_begin list = - if debug_present_chunks then +- b.block_num +- s.s_verified_bitmap.[b.block_num] +- b.block_begin b.block_end +- (match s.s_blocks.(b.block_num) with +- EmptyBlock -> "empty" +- | PartialBlock _ -> "partial" +- | CompleteBlock -> "complete" +- | VerifiedBlock -> "verified" +- ); +- let br = b.block_ranges in +- lprintf " first range: %Ld(%Ld)" +- br.range_begin +- (br.range_end -- br.range_current_begin); +- lprint_newline (); +- ) up.up_ranges; +- end; +- if !exit_on_error then exit 2 +- end else +- if string_length > 0 then +- match s.s_networks with +- [] -> assert false +- | t :: _ when t.t_primary -> +- file_write t.t_file +- r.range_current_begin +- str string_pos string_length; +- | _ -> () +- end; +- range_received (Some t) r r.range_current_begin file_end; +- +- end +- ) up.up_ranges; +- clean_ranges up +- with e -> +- raise e +- +- +-(*************************************************************************) +-(* *) +-(* present_chunks *) +-(* *) +-(*************************************************************************) +- +-let present_chunks s = +- let nblocks = Array.length s.s_blocks in +-(* lprintf "present_chunks...%d blocks\n" nblocks; *) +- +- let rec iter_block_out i block_begin list = +- if debug_present_chunks then - lprintf_nl () "iter_block_out %d bb: %Ld" -+ lprintf_nl "iter_block_out %d bb: %Ld" - i block_begin; - - if i = nblocks then List.rev list else -@@ -2424,7 +2391,7 @@ - - and iter_block_in i block_begin chunk_begin list = - if debug_present_chunks then +- i block_begin; +- +- if i = nblocks then List.rev list else +- let block_end = compute_block_end s i in +- match s.s_blocks.(i) with +- EmptyBlock -> +- iter_block_out (i+1) block_end list +- | CompleteBlock | VerifiedBlock -> +- let block_begin = compute_block_begin s i in +- iter_block_in (i+1) block_end block_begin list +- | PartialBlock b -> +- iter_range_out i block_end block_begin b.block_ranges list +- +- and iter_block_in i block_begin chunk_begin list = +- if debug_present_chunks then - lprintf_nl () "iter_block_in %d bb: %Ld cb:%Ld" -+ lprintf_nl "iter_block_in %d bb: %Ld cb:%Ld" - i block_begin chunk_begin - ; - -@@ -2444,7 +2411,7 @@ - - and iter_range_out i block_end block_begin r list = - if debug_present_chunks then +- i block_begin chunk_begin +- ; +- +- if i = nblocks then +- let list = (chunk_begin, s.s_size) :: list in +- List.rev list +- else +- let block_end = compute_block_end s i in +- match s.s_blocks.(i) with +- EmptyBlock -> +- iter_block_out (i+1) block_end ( (chunk_begin, block_begin) :: list) +- | CompleteBlock | VerifiedBlock -> +- iter_block_in (i+1) block_end chunk_begin list +- | PartialBlock b -> +- iter_range_in i block_end +- chunk_begin block_begin b.block_ranges list ++ match file_state t.t_file with ++ | FilePaused ++ | FileAborted _ ++ | FileCancelled ++ | FileShared ++ | FileNew ++ | FileDownloaded -> ++ if !verbose then ++ lprintf_nl "CommonSwarming.received: wrong file state"; ++ () ++ | FileDownloading ++ | FileQueued -> ++ try ++ List.iter (fun (_,_,r) -> ++ (* was: r.range_begin < file_end && r.range_end > file_begin *) ++ if r.range_begin >= file_begin && ++ r.range_begin < file_end then ++ let file_end = min file_end r.range_end in ++ let written_len = file_end -- r.range_begin in ++ let string_pos = string_begin + ++ Int64.to_int (r.range_begin -- file_begin) in ++ let string_length = Int64.to_int written_len in ++ (* None of those conditions can happen anymore *) ++(* if string_pos < 0 || ++ string_pos < string_begin || ++ string_len < string_length then ++ debug_bad_write r string_pos ++ else *) ++ if string_length > 0 then ++ match s.s_networks with ++ | [] -> assert false ++ | tprim :: _ -> ++ assert (tprim.t_primary); ++ file_write tprim.t_file ++ r.range_begin ++ str string_pos string_length; ++ range_received (Some t) r r.range_begin file_end; ++ ) up.up_ranges; ++ remove_completed_uploader_ranges up ++ with e -> ++ remove_completed_uploader_ranges up; ++ raise e + +- and iter_range_out i block_end block_begin r list = +- if debug_present_chunks then - lprintf_nl () "iter_range_out %d nb: %Ld bb:%Ld" -+ lprintf_nl "iter_range_out %d nb: %Ld bb:%Ld" - i block_end block_begin; - - if r.range_begin > block_begin then -@@ -2476,7 +2443,7 @@ - - and iter_range_in i block_end chunk_begin chunk_end r list = - if debug_present_chunks then +- i block_end block_begin; ++(** compute the list of present intervals of a swarmer *) + +- if r.range_begin > block_begin then +- iter_range_in i block_end block_begin r.range_begin r list ++let present_intervals s = ++ (* intervals is a reversed list of intervals *) ++ let append_interval ((interval_begin, interval_end) as interval) intervals = ++ (* remove void intervals *) ++ if interval_begin = interval_end then intervals + else ++ match intervals with ++ | [] -> [interval] ++ | (last_interval_begin, last_interval_end) :: other_intervals -> ++ if last_interval_end < interval_begin then ++ interval :: intervals ++ else ++ (* coalescing intervals *) ++ (last_interval_begin, interval_end) :: other_intervals in + +- if r.range_current_begin > block_begin then begin +- if r.range_current_begin < r.range_end then +- let list = (r.range_begin, r.range_current_begin) :: list in +- match r.range_next with +- None -> +- iter_block_out (i+1) block_end list +- | Some rr -> +- iter_range_out i block_end r.range_end rr list +- else +- match r.range_next with +- None -> +- iter_block_in (i+1) block_end r.range_begin list +- | Some rr -> +- iter_range_in i block_end +- r.range_begin r.range_end rr list +- end else +- match r.range_next with +- None -> +- iter_block_out (i+1) block_end list +- | Some rr -> +- iter_range_out i block_end r.range_end rr list +- +- +- and iter_range_in i block_end chunk_begin chunk_end r list = +- if debug_present_chunks then - lprintf_nl () "iter_range_in %d bn: %Ld cb:%Ld ce: %Ld" -+ lprintf_nl "iter_range_in %d bn: %Ld cb:%Ld ce: %Ld" - i block_end chunk_begin chunk_end; +- i block_end chunk_begin chunk_end; +- +- if r.range_current_begin < r.range_end then +- let list = (chunk_begin, r.range_current_begin) :: list in +- match r.range_next with +- None -> +- iter_block_out (i+1) block_end list +- | Some rr -> +- iter_range_out i block_end r.range_end rr list +- else +- match r.range_next with +- None -> +- iter_block_in (i+1) block_end chunk_begin list +- | Some rr -> +- iter_range_in i block_end chunk_begin r.range_end rr list +- in +- let chunks = iter_block_out 0 zero [] in +-(* lprintf "present_chunks done\n"; *) +- chunks ++ List.rev ( ++ array_fold_lefti (fun acc i b -> ++ match s.s_blocks.(i) with ++ | EmptyBlock -> acc ++ | CompleteBlock | VerifiedBlock -> ++ append_interval (compute_block_begin s i, compute_block_end s i) acc ++ | PartialBlock b -> ++ let acc, last_interval_end = ++ block_ranges_fold (fun (acc, lie) r -> ++ (append_interval (lie, r.range_begin) acc, r.range_end) ++ ) (acc, compute_block_begin s i) b in ++ append_interval (last_interval_end, compute_block_end s i) acc ++ ) [] s.s_blocks) - if r.range_current_begin < r.range_end then -@@ -2550,7 +2517,7 @@ + (*************************************************************************) + (* *) +@@ -2550,7 +2285,7 @@ sw := (t, i, pos) :: !sw; iter (i+1) len (pos ++ bsize) bsize size in @@ -1471,7 +3402,7 @@ ) swarmers_by_num; Hashtbl.iter (fun c (has, has_not) -> match !has, !has_not with -@@ -2600,7 +2567,7 @@ +@@ -2600,7 +2335,7 @@ match bitmap.[i] with | '2' -> if t.t_converted_verified_bitmap.[i] < '2' then begin @@ -1480,7 +3411,7 @@ t.t_converted_verified_bitmap.[i] <- '2' end -@@ -2625,7 +2592,7 @@ +@@ -2625,7 +2360,7 @@ () ) t.t_blocks_of_chunk.(i); if t.t_converted_verified_bitmap.[i] <> '3' then @@ -1489,7 +3420,16 @@ | _ -> () done -@@ -2794,7 +2761,7 @@ +@@ -2746,7 +2481,7 @@ + + let set_present t = set_present t.t_s + let set_absent t = set_absent t.t_s +-let present_chunks t = present_chunks t.t_s ++let present_intervals t = present_intervals t.t_s + let print_t str t = print_s str t.t_s + let print_uploaders t = print_uploaders t.t_s + +@@ -2794,7 +2529,7 @@ (get_value "file_all_chunks" value_to_string) with e -> @@ -1498,7 +3438,7 @@ (Printexc2.to_string e); ); -@@ -2804,7 +2771,7 @@ +@@ -2804,7 +2539,7 @@ *) if primary then begin @@ -1507,7 +3447,7 @@ let present = try let present = (get_value "file_present_chunks" -@@ -2813,46 +2780,46 @@ +@@ -2813,46 +2548,46 @@ set_present t present; present with e -> @@ -1547,8 +3487,9 @@ + lprintf_nl " (%Ld,%Ld);" x y ) present; - let p = present_chunks t in +- let p = present_chunks t in - lprintf_nl () "ERROR: present now:"; ++ let p = present_intervals t in + lprintf_nl "ERROR: present now:"; let total = ref zero in @@ -1566,7 +3507,16 @@ end; if !exit_on_error then exit 2 end -@@ -2955,7 +2922,7 @@ +@@ -2893,7 +2628,7 @@ + ("file_present_chunks", List + (List.map (fun (i1,i2) -> + SmallList [int64_to_value i1; int64_to_value i2]) +- (present_chunks t))) :: ++ (present_intervals t))) :: + ("file_downloaded", int64_to_value (downloaded t)) :: + + ("file_chunks_age", List (Array.to_list +@@ -2955,27 +2690,28 @@ let s1 = HS.find swarmers_by_name { dummy_swarmer with s_filename = file_disk_name f1 } in let s2 = HS.find swarmers_by_name { dummy_swarmer with s_filename = file_disk_name f2 } in @@ -1575,16 +3525,34 @@ failwith "Files are already sharing their swarmer"; if s1.s_size <> s2.s_size then -@@ -2964,7 +2931,7 @@ - let t2 = match s2.s_networks with - [t] -> t + failwith "Files don't have the same size"; + +- let t2 = match s2.s_networks with +- [t] -> t ++ let t2 = ++ match s2.s_networks with ++ | [t] -> t | list -> - lprintf_nl () "s_networks: %d files" (List.length list); + lprintf_nl "s_networks: %d files" (List.length list); failwith "Second file is already merged with other files" in -@@ -3012,7 +2979,7 @@ + let t1 = + match s1.s_networks with +- [] -> assert false ++ | [] -> assert false + | t1 :: _ -> + match t1.t_verifier with +- NoVerification -> ++ | NoVerification | VerificationNotAvailable -> + failwith "Cannot use first file as a primary for swarming (no verification scheme)" +- | _ -> t1 ++ | Verification _ | ForceVerification -> t1 + in + + begin +@@ -3012,7 +2748,7 @@ None -> () | Some sw -> if not (has_secondaries sw) then HS.remove swarmers_by_name sw.t_s @@ -1593,7 +3561,7 @@ (*************************************************************************) (* *) -@@ -3037,7 +3004,7 @@ +@@ -3037,7 +2773,7 @@ let get_value name conv = conv (List.assoc name assocs) in let file_size = get_value "file_size" value_to_int64 in let file_name = get_value "file_name" value_to_string in @@ -1602,7 +3570,7 @@ let block_sizes = get_value "file_chunk_sizes" (value_to_list value_to_int64) in List.iter (fun bsize -> -@@ -3053,7 +3020,7 @@ +@@ -3053,7 +2789,7 @@ ("file_name", string_to_value s.s_filename); ("file_bitmap", string_to_value s.s_verified_bitmap); ("file_chunk_sizes", list_to_value int64_to_value @@ -1611,16 +3579,19 @@ ] let t = -@@ -3190,7 +3157,7 @@ +@@ -3189,9 +2925,8 @@ + Array.length up.up_complete_blocks * 4 + List.length up.up_ranges * (12 + 16 + 12 + 12 + 4) + Array.length up.up_partial_blocks * (16 + 12 + 12) + - (8 + match up.up_chunks with +- (8 + match up.up_chunks with - AvailableRanges list -> List.length list * (12 + 12 + 12 + 12) -+ AvailableIntervals list -> List.length list * (12 + 12 + 12 + 12) - | AvailableCharBitmap s -> 8 + String.length s +- | AvailableCharBitmap s -> 8 + String.length s ++ (8 + match up.up_intervals with ++ | AvailableIntervals list -> List.length list * (12 + 12 + 12 + 12) | AvailableBitv b -> let ws = Sys.word_size in (ws/8) + ((ws / 8) * (Bitv.length b / (ws - 2))) ) ; -@@ -3211,7 +3178,7 @@ + incr counter; +@@ -3211,7 +2946,7 @@ if bitmap.[i] <> '3' then raise Not_found; done; if file_size file <> downloaded t then diff --git a/net-p2p/mldonkey-devel/files/patch-src__networks__bittorrent__bTClients.ml b/net-p2p/mldonkey-devel/files/patch-src__networks__bittorrent__bTClients.ml index a2f71c0..32d17df 100644 --- a/net-p2p/mldonkey-devel/files/patch-src__networks__bittorrent__bTClients.ml +++ b/net-p2p/mldonkey-devel/files/patch-src__networks__bittorrent__bTClients.ml @@ -1,5 +1,5 @@ --- ./src/networks/bittorrent/bTClients.ml.orig Sat Apr 8 21:26:40 2006 -+++ ./src/networks/bittorrent/bTClients.ml Thu Apr 20 11:04:03 2006 ++++ ./src/networks/bittorrent/bTClients.ml Sun May 7 06:39:10 2006 @@ -561,7 +561,7 @@ match c.client_uploader with None -> @@ -9,3 +9,12 @@ c.client_uploader <- Some up; up | Some up -> +@@ -581,7 +581,7 @@ + let chunks = c.client_new_chunks in + c.client_new_chunks <- []; + List.iter (fun n -> Bitv.set bitmap n true) chunks; +- CommonSwarming.update_uploader up (AvailableBitv bitmap); ++ CommonSwarming.update_uploader_intervals up (AvailableBitv bitmap); + end + + diff --git a/net-p2p/mldonkey-devel/files/patch-src__networks__bittorrent__bTGlobals.ml b/net-p2p/mldonkey-devel/files/patch-src__networks__bittorrent__bTGlobals.ml index 102d6d5..ea6fd3a 100644 --- a/net-p2p/mldonkey-devel/files/patch-src__networks__bittorrent__bTGlobals.ml +++ b/net-p2p/mldonkey-devel/files/patch-src__networks__bittorrent__bTGlobals.ml @@ -1,5 +1,5 @@ --- ./src/networks/bittorrent/bTGlobals.ml.orig Sat Apr 8 21:26:40 2006 -+++ ./src/networks/bittorrent/bTGlobals.ml Thu Apr 20 11:04:03 2006 ++++ ./src/networks/bittorrent/bTGlobals.ml Sun May 7 06:39:10 2006 @@ -252,8 +252,7 @@ else set_trackers file [t.torrent_announce]; diff --git a/net-p2p/mldonkey-devel/files/patch-src__networks__bittorrent__bTInteractive.ml b/net-p2p/mldonkey-devel/files/patch-src__networks__bittorrent__bTInteractive.ml new file mode 100644 index 0000000..042bd25 --- /dev/null +++ b/net-p2p/mldonkey-devel/files/patch-src__networks__bittorrent__bTInteractive.ml @@ -0,0 +1,11 @@ +--- ./src/networks/bittorrent/bTInteractive.ml.orig Sat Apr 8 21:26:40 2006 ++++ ./src/networks/bittorrent/bTInteractive.ml Sun May 7 06:39:10 2006 +@@ -372,7 +372,7 @@ + None -> + lprintf_nl () "verify_chunks: no swarmer to verify chunks" + | Some swarmer -> +- CommonSwarming.verify_all_chunks swarmer true ++ CommonSwarming.verify_all_chunks_immediately swarmer + + let remove_all_clients file = + Hashtbl.clear file.file_clients; diff --git a/net-p2p/mldonkey-devel/files/patch-src__networks__donkey__donkeyGlobals.ml b/net-p2p/mldonkey-devel/files/patch-src__networks__donkey__donkeyGlobals.ml index 89ca630..d77815e 100644 --- a/net-p2p/mldonkey-devel/files/patch-src__networks__donkey__donkeyGlobals.ml +++ b/net-p2p/mldonkey-devel/files/patch-src__networks__donkey__donkeyGlobals.ml @@ -1,5 +1,5 @@ --- ./src/networks/donkey/donkeyGlobals.ml.orig Sat Apr 8 21:26:40 2006 -+++ ./src/networks/donkey/donkeyGlobals.ml Thu Apr 20 11:04:03 2006 ++++ ./src/networks/donkey/donkeyGlobals.ml Sun May 7 06:39:10 2006 @@ -397,7 +397,7 @@ (match file_state with FileShared -> () diff --git a/net-p2p/mldonkey-devel/files/patch-src__networks__donkey__donkeyInteractive.ml b/net-p2p/mldonkey-devel/files/patch-src__networks__donkey__donkeyInteractive.ml new file mode 100644 index 0000000..eba8b14 --- /dev/null +++ b/net-p2p/mldonkey-devel/files/patch-src__networks__donkey__donkeyInteractive.ml @@ -0,0 +1,20 @@ +--- ./src/networks/donkey/donkeyInteractive.ml.orig Sat Apr 8 21:26:40 2006 ++++ ./src/networks/donkey/donkeyInteractive.ml Sun May 7 06:39:10 2006 +@@ -507,7 +507,7 @@ + match file.file_swarmer with + None -> () + | Some swarmer -> +- CommonSwarming.verify_all_chunks swarmer false ++ CommonSwarming.verify_all_chunks swarmer + + (* + if file.file_chunks <> [||] then +@@ -613,7 +613,7 @@ + match file.file_swarmer with + None -> () + | Some swarmer -> +- CommonSwarming.verify_all_chunks swarmer true ++ CommonSwarming.verify_all_chunks_immediately swarmer + + let register_commands list = + register_commands diff --git a/net-p2p/mldonkey-devel/files/patch-src__networks__donkey__donkeyOneFile.ml b/net-p2p/mldonkey-devel/files/patch-src__networks__donkey__donkeyOneFile.ml new file mode 100644 index 0000000..4c63704 --- /dev/null +++ b/net-p2p/mldonkey-devel/files/patch-src__networks__donkey__donkeyOneFile.ml @@ -0,0 +1,21 @@ +--- ./src/networks/donkey/donkeyOneFile.ml.orig Sat Apr 8 21:26:40 2006 ++++ ./src/networks/donkey/donkeyOneFile.ml Sun May 7 06:39:10 2006 +@@ -217,7 +217,7 @@ + (f, chunks, up) :: tail -> + if f != file then iter tail + else begin +- CommonSwarming.update_uploader up ++ CommonSwarming.update_uploader_intervals up + (AvailableBitv client_chunks); + Bitv.blit client_chunks 0 chunks 0 (Bitv.length chunks) + end +@@ -239,8 +239,7 @@ + match c.client_download with + None -> () + | Some (file, up) -> +- CommonSwarming.clear_uploader_block up; +- CommonSwarming.clear_uploader_ranges up; ++ CommonSwarming.unregister_uploader up; + c.client_download <- None + + let send_get_range_request c file ranges = diff --git a/net-p2p/mldonkey-devel/files/patch-src__networks__fasttrack__fasttrackGlobals.ml b/net-p2p/mldonkey-devel/files/patch-src__networks__fasttrack__fasttrackGlobals.ml index ac6d26d..bc8f3c9 100644 --- a/net-p2p/mldonkey-devel/files/patch-src__networks__fasttrack__fasttrackGlobals.ml +++ b/net-p2p/mldonkey-devel/files/patch-src__networks__fasttrack__fasttrackGlobals.ml @@ -1,5 +1,5 @@ --- ./src/networks/fasttrack/fasttrackGlobals.ml.orig Sat Apr 8 21:26:41 2006 -+++ ./src/networks/fasttrack/fasttrackGlobals.ml Thu Apr 20 11:04:04 2006 ++++ ./src/networks/fasttrack/fasttrackGlobals.ml Sun May 7 06:39:10 2006 @@ -298,7 +298,7 @@ } in diff --git a/net-p2p/mldonkey-devel/files/patch-src__networks__fileTP__fileTPClients.ml b/net-p2p/mldonkey-devel/files/patch-src__networks__fileTP__fileTPClients.ml index 59f80b1..782bdd8 100644 --- a/net-p2p/mldonkey-devel/files/patch-src__networks__fileTP__fileTPClients.ml +++ b/net-p2p/mldonkey-devel/files/patch-src__networks__fileTP__fileTPClients.ml @@ -1,5 +1,5 @@ --- ./src/networks/fileTP/fileTPClients.ml.orig Sat Apr 8 21:26:41 2006 -+++ ./src/networks/fileTP/fileTPClients.ml Thu Apr 20 11:04:04 2006 ++++ ./src/networks/fileTP/fileTPClients.ml Sun May 7 06:39:10 2006 @@ -269,7 +269,7 @@ let chunks = [ Int64.zero, file_size file ] in let up = CommonSwarming.register_uploader swarmer diff --git a/net-p2p/mldonkey-devel/files/patch-src__networks__fileTP__fileTPGlobals.ml b/net-p2p/mldonkey-devel/files/patch-src__networks__fileTP__fileTPGlobals.ml index acff5b2..1157c2b 100644 --- a/net-p2p/mldonkey-devel/files/patch-src__networks__fileTP__fileTPGlobals.ml +++ b/net-p2p/mldonkey-devel/files/patch-src__networks__fileTP__fileTPGlobals.ml @@ -1,5 +1,5 @@ --- ./src/networks/fileTP/fileTPGlobals.ml.orig Mon Apr 10 16:16:13 2006 -+++ ./src/networks/fileTP/fileTPGlobals.ml Thu Apr 20 11:04:04 2006 ++++ ./src/networks/fileTP/fileTPGlobals.ml Sun May 7 06:39:10 2006 @@ -120,7 +120,7 @@ in file.file_file.impl_file_size <- size; diff --git a/net-p2p/mldonkey-devel/files/patch-src__networks__gnutella__gnutellaClients.ml b/net-p2p/mldonkey-devel/files/patch-src__networks__gnutella__gnutellaClients.ml index 08f39cd..4114d5b 100644 --- a/net-p2p/mldonkey-devel/files/patch-src__networks__gnutella__gnutellaClients.ml +++ b/net-p2p/mldonkey-devel/files/patch-src__networks__gnutella__gnutellaClients.ml @@ -1,5 +1,5 @@ --- ./src/networks/gnutella/gnutellaClients.ml.orig Sat Apr 8 21:26:41 2006 -+++ ./src/networks/gnutella/gnutellaClients.ml Thu Apr 20 11:04:04 2006 ++++ ./src/networks/gnutella/gnutellaClients.ml Sun May 7 06:39:10 2006 @@ -479,7 +479,7 @@ let chunks = [ Int64.zero, file_size file ] in let up = CommonSwarming.register_uploader swarmer diff --git a/net-p2p/mldonkey-devel/files/patch-src__networks__gnutella__gnutellaComplexOptions.ml b/net-p2p/mldonkey-devel/files/patch-src__networks__gnutella__gnutellaComplexOptions.ml new file mode 100644 index 0000000..5741b61 --- /dev/null +++ b/net-p2p/mldonkey-devel/files/patch-src__networks__gnutella__gnutellaComplexOptions.ml @@ -0,0 +1,11 @@ +--- ./src/networks/gnutella/gnutellaComplexOptions.ml.orig Sat Apr 8 21:26:41 2006 ++++ ./src/networks/gnutella/gnutellaComplexOptions.ml Sun May 7 06:39:10 2006 +@@ -183,7 +183,7 @@ + (* "file_present_chunks", List + (List.map (fun (i1,i2) -> + SmallList [int64_to_value i1; int64_to_value i2]) +- (CommonSwarming.present_chunks file.file_swarmer)); ++ (CommonSwarming.present_intervals file.file_swarmer)); + *) + ] + in diff --git a/net-p2p/mldonkey-devel/files/patch-src__networks__gnutella__gnutellaGlobals.ml b/net-p2p/mldonkey-devel/files/patch-src__networks__gnutella__gnutellaGlobals.ml index 542accb..ae3de29 100644 --- a/net-p2p/mldonkey-devel/files/patch-src__networks__gnutella__gnutellaGlobals.ml +++ b/net-p2p/mldonkey-devel/files/patch-src__networks__gnutella__gnutellaGlobals.ml @@ -1,5 +1,5 @@ --- ./src/networks/gnutella/gnutellaGlobals.ml.orig Sat Apr 8 21:26:41 2006 -+++ ./src/networks/gnutella/gnutellaGlobals.ml Thu Apr 20 11:04:04 2006 ++++ ./src/networks/gnutella/gnutellaGlobals.ml Sun May 7 06:39:10 2006 @@ -325,8 +325,7 @@ in if !verbose then diff --git a/net-p2p/mldonkey-devel/files/patch-src__networks__gnutella__gnutellaInteractive.ml b/net-p2p/mldonkey-devel/files/patch-src__networks__gnutella__gnutellaInteractive.ml new file mode 100644 index 0000000..10805c7 --- /dev/null +++ b/net-p2p/mldonkey-devel/files/patch-src__networks__gnutella__gnutellaInteractive.ml @@ -0,0 +1,11 @@ +--- ./src/networks/gnutella/gnutellaInteractive.ml.orig Sat Apr 8 21:26:41 2006 ++++ ./src/networks/gnutella/gnutellaInteractive.ml Sun May 7 06:39:10 2006 +@@ -252,7 +252,7 @@ + match file.file_ttr with + None -> failwith "No TTR for verification" + | Some ttt -> +- CommonSwarming.verify_all_chunks swarmer true ++ CommonSwarming.verify_all_chunks_immediately swarmer + ); + + file_ops.op_file_recover <- (fun file -> |