add lexproduct, lexicographic-order product iterator

Makefile

@@ -198,7 +198,6 @@ CORE_SRCS := $(addprefix $(JULIAHOME)/, \
 		base/inference.jl \
 		base/int.jl \
 		base/intset.jl \
-		base/iterator.jl \
 		base/nofloat_hashing.jl \
 		base/number.jl \
 		base/operators.jl \

base/REPLCompletions.jl

@@ -376,10 +376,10 @@ function bslash_completions(string, pos)
         # return possible matches; these cannot be mixed with regular
         # Julian completions as only latex / emoji symbols contain the leading \
         if startswith(s, "\\:") # emoji
-            emoji_names = filter(k -> startswith(k, s), keys(emoji_symbols))
+            emoji_names = IterTools.filter(k -> startswith(k, s), keys(emoji_symbols))
             return (true, (sort!(collect(emoji_names)), slashpos:pos, true))
         else # latex
-            latex_names = filter(k -> startswith(k, s), keys(latex_symbols))
+            latex_names = IterTools.filter(k -> startswith(k, s), keys(latex_symbols))
             return (true, (sort!(collect(latex_names)), slashpos:pos, true))
         end
     end

base/asyncmap.jl

@@ -1,5 +1,6 @@
 # This file is a part of Julia. License is MIT: http://julialang.org/license
 
+using Base.IterTools.Enumerate
 
 """
     AsyncCollector(f, results, c...; ntasks=0) -> iterator

base/broadcast.jl

@@ -226,7 +226,7 @@ end
             else
                 R = typejoin(eltype(B), S)
                 new = similar(B, R)
-                for II in take(iter, count)
+                for II in IterTools.take(iter, count)
                     new[II] = B[II]
                 end
                 new[I] = V

base/client.jl

@@ -23,7 +23,7 @@ end
 # Create a docstring with an automatically generated list
 # of colors.
 const possible_formatting_symbols = [:normal, :bold]
-available_text_colors = collect(filter(x -> !isa(x, Integer), keys(text_colors)))
+available_text_colors = collect(IterTools.filter(x -> !isa(x, Integer), keys(text_colors)))
 available_text_colors = cat(1,
     sort(intersect(available_text_colors, possible_formatting_symbols), rev=true),
     sort(setdiff(  available_text_colors, possible_formatting_symbols)))

base/coreimg.jl

@@ -61,7 +61,6 @@ include("reduce.jl")
 ## core structures
 include("intset.jl")
 include("dict.jl")
-include("iterator.jl")
 
 # core docsystem
 include("docs/core.jl")

base/dates/Dates.jl

@@ -4,6 +4,8 @@ module Dates
 
 importall ..Base.Operators
 
+using Base.IterTools
+
 include("types.jl")
 include("periods.jl")
 include("accessors.jl")

base/deprecated.jl

@@ -1019,4 +1019,14 @@ eval(Multimedia, :(macro textmime(mime)
     end
 end))
 
+@deprecate_binding Filter    IterTools.Filter
+@deprecate_binding Zip       IterTools.Zip
+@deprecate filter(flt, itr)  IterTools.filter(flt, itr)
+@deprecate_binding rest      IterTools.rest
+@deprecate_binding countfrom IterTools.countfrom
+@deprecate_binding take      IterTools.take
+@deprecate_binding drop      IterTools.drop
+@deprecate_binding cycle     IterTools.cycle
+@deprecate_binding repeated  IterTools.repeated
+
 # End deprecations scheduled for 0.6

base/essentials.jl

@@ -63,6 +63,12 @@ function tuple_type_tail(T::DataType)
     return Tuple{argtail(T.parameters...)...}
 end
 
+tuple_type_cons{S}(::Type{S}, ::Type{Union{}}) = Union{}
+function tuple_type_cons{S,T<:Tuple}(::Type{S}, ::Type{T})
+    @_pure_meta
+    Tuple{S, T.parameters...}
+end
+
 isvarargtype(t::ANY) = isa(t, DataType) && is((t::DataType).name, Vararg.name)
 isvatuple(t::DataType) = (n = length(t.parameters); n > 0 && isvarargtype(t.parameters[n]))
 unwrapva(t::ANY) = isvarargtype(t) ? t.parameters[1] : t
@@ -231,3 +237,5 @@ function vector_any(xs::ANY...)
     end
     a
 end
+
+isempty(itr) = done(itr, start(itr))

base/exports.jl

@@ -23,6 +23,7 @@ export
     Docs,
     Markdown,
     Threads,
+    IterTools,
 
 # Types
     AbstractChannel,
@@ -60,7 +61,6 @@ export
     Enumerate,
     Factorization,
     FileMonitor,
-    Filter,
     FloatRange,
     Future,
     Hermitian,
@@ -123,7 +123,6 @@ export
     VersionNumber,
     WeakKeyDict,
     WorkerConfig,
-    Zip,
 
 # Ccall types
     Cchar,
@@ -958,16 +957,11 @@ export
 
 # iteration
     done,
-    enumerate,
     next,
     start,
+
+    enumerate,  # re-exported from IterTools
     zip,
-    rest,
-    countfrom,
-    take,
-    drop,
-    cycle,
-    repeated,
 
 # object identity and equality
     copy,

base/generator.jl

@@ -55,11 +55,6 @@ result, and algorithms that resize their result incrementally.
 iteratorsize(x) = iteratorsize(typeof(x))
 iteratorsize(::Type) = HasLength()  # HasLength is the default
 
-and_iteratorsize{T}(isz::T, ::T) = isz
-and_iteratorsize(::HasLength, ::HasShape) = HasLength()
-and_iteratorsize(::HasShape, ::HasLength) = HasLength()
-and_iteratorsize(a, b) = SizeUnknown()
-
 abstract IteratorEltype
 immutable EltypeUnknown <: IteratorEltype end
 immutable HasEltype <: IteratorEltype end
@@ -81,9 +76,6 @@ values.
 iteratoreltype(x) = iteratoreltype(typeof(x))
 iteratoreltype(::Type) = HasEltype()  # HasEltype is the default
 
-and_iteratoreltype{T}(iel::T, ::T) = iel
-and_iteratoreltype(a, b) = EltypeUnknown()
-
 iteratorsize{T<:AbstractArray}(::Type{T}) = HasShape()
 iteratorsize{I,F}(::Type{Generator{I,F}}) = iteratorsize(I)
 length(g::Generator) = length(g.iter)

base/inference.jl

@@ -1017,7 +1017,8 @@ function abstract_apply(af::ANY, fargs, aargtypes::Vector{Any}, vtypes::VarTable
 end
 
 function pure_eval_call(f::ANY, argtypes::ANY, atype::ANY, vtypes::VarTable, sv::InferenceState)
-    for a in drop(argtypes,1)
+    for i = 2:length(argtypes)
+        a = argtypes[i]
         if !(isa(a,Const) || isconstType(a,false))
             return false
         end
@@ -1053,7 +1054,7 @@ function pure_eval_call(f::ANY, argtypes::ANY, atype::ANY, vtypes::VarTable, sv:
         return false
     end
 
-    args = Any[ isa(a,Const) ? a.val : a.parameters[1] for a in drop(argtypes,1) ]
+    args = Any[ (a=argtypes[i]; isa(a,Const) ? a.val : a.parameters[1]) for i in 2:length(argtypes) ]
     try
         return abstract_eval_constant(f(args...))
     catch

base/iterator.jl → base/itertools.jl

@@ -1,6 +1,14 @@
 # This file is a part of Julia. License is MIT: http://julialang.org/license
 
-isempty(itr) = done(itr, start(itr))
+module IterTools
+
+global Filter
+
+import Base: start, done, next, isempty, length, size, eltype, iteratorsize, iteratoreltype, indices, ndims
+
+using Base: tuple_type_cons, SizeUnknown, HasLength, HasShape, IsInfinite, EltypeUnknown, HasEltype, OneTo
+
+export enumerate, zip, rest, countfrom, take, drop, cycle, repeated, product, lexproduct, flatten, partition
 
 _min_length(a, b, ::IsInfinite, ::IsInfinite) = min(length(a),length(b)) # inherit behaviour, error
 _min_length(a, b, A, ::IsInfinite) = length(a)
@@ -12,6 +20,14 @@ _diff_length(a, b, ::IsInfinite, ::IsInfinite) = 0
 _diff_length(a, b, ::IsInfinite, B) = length(a) # inherit behaviour, error
 _diff_length(a, b, A, B) = max(length(a)-length(b), 0)
 
+and_iteratorsize{T}(isz::T, ::T) = isz
+and_iteratorsize(::HasLength, ::HasShape) = HasLength()
+and_iteratorsize(::HasShape, ::HasLength) = HasLength()
+and_iteratorsize(a, b) = SizeUnknown()
+
+and_iteratoreltype{T}(iel::T, ::T) = iel
+and_iteratoreltype(a, b) = EltypeUnknown()
+
 # enumerate
 
 immutable Enumerate{I}
@@ -143,11 +159,6 @@ zip(a, b, c...) = Zip(a, zip(b, c...))
 length(z::Zip) = _min_length(z.a, z.z, iteratorsize(z.a), iteratorsize(z.z))
 size(z::Zip) = promote_shape(size(z.a), size(z.z))
 indices(z::Zip) = promote_shape(indices(z.a), indices(z.z))
-tuple_type_cons{S}(::Type{S}, ::Type{Union{}}) = Union{}
-function tuple_type_cons{S,T<:Tuple}(::Type{S}, ::Type{T})
-    @_pure_meta
-    Tuple{S, T.parameters...}
-end
 eltype{I,Z}(::Type{Zip{I,Z}}) = tuple_type_cons(eltype(I), eltype(Z))
 @inline start(z::Zip) = tuple(start(z.a), start(z.z))
 @inline function next(z::Zip, st)
@@ -245,42 +256,6 @@ rest_iteratorsize(::IsInfinite) = IsInfinite()
 iteratorsize{I,S}(::Type{Rest{I,S}}) = rest_iteratorsize(iteratorsize(I))
 
 
-"""
-    head_and_tail(c, n) -> head, tail
-
-Returns `head`: the first `n` elements of `c`;
-and `tail`: an iterator over the remaining elements.
-
-```jldoctest
-julia> a = 1:10
-1:10
-
-julia> b, c = Base.head_and_tail(a, 3)
-([1,2,3],Base.Rest{UnitRange{Int64},Int64}(1:10,4))
-
-julia> collect(c)
-7-element Array{Any,1}:
-  4
-  5
-  6
-  7
-  8
-  9
- 10
-```
-"""
-function head_and_tail(c, n)
-    head = Vector{eltype(c)}(n)
-    s = start(c)
-    i = 0
-    while i < n && !done(c, s)
-        i += 1
-        head[i], s = next(c, s)
-    end
-    return resize!(head, i), rest(c, s)
-end
-
-
 # Count -- infinite counting
 
 immutable Count{S<:Number}
@@ -521,6 +496,13 @@ _prod_indices(a, b, ::HasShape,  ::HasShape)   = (indices(a)..., indices(b)...)
 _prod_indices(a, b, A, B) =
     throw(ArgumentError("Cannot construct indices for objects of types $(typeof(a)) and $(typeof(b))"))
 
+prod_iteratorsize(::Union{HasLength,HasShape}, ::Union{HasLength,HasShape}) = HasShape()
+# products can have an infinite iterator
+prod_iteratorsize(::IsInfinite, ::IsInfinite) = IsInfinite()
+prod_iteratorsize(a, ::IsInfinite) = IsInfinite()
+prod_iteratorsize(::IsInfinite, b) = IsInfinite()
+prod_iteratorsize(a, b) = SizeUnknown()
+
 # one iterator
 immutable Prod1{I} <: AbstractProdIterator
     a::I
@@ -555,7 +537,7 @@ a tuple whose `i`th element comes from the `i`th argument iterator. The first it
 changes the fastest. Example:
 
 ```jldoctest
-julia> collect(product(1:2,3:5))
+julia> vec(collect(product(1:2,3:5)))
 6-element Array{Tuple{Int64,Int64},1}:
  (1,3)
  (2,3)
@@ -614,12 +596,75 @@ iteratorsize{I1,I2}(::Type{Prod{I1,I2}}) = prod_iteratorsize(iteratorsize(I1),it
     ((x[1][1],x[1][2]...), x[2])
 end
 
-prod_iteratorsize(::Union{HasLength,HasShape}, ::Union{HasLength,HasShape}) = HasShape()
-# products can have an infinite iterator
-prod_iteratorsize(::IsInfinite, ::IsInfinite) = IsInfinite()
-prod_iteratorsize(a, ::IsInfinite) = IsInfinite()
-prod_iteratorsize(::IsInfinite, b) = IsInfinite()
-prod_iteratorsize(a, b) = SizeUnknown()
+# lexicographic order product iterator
+
+immutable LexProd2{I1, I2} <: AbstractProdIterator
+    a::I1
+    b::I2
+end
+
+"""
+    lexproduct(iters...)
+
+Returns an iterator over the product of several iterators. Each generated element is
+a tuple whose `i`th element comes from the `i`th argument iterator. The first iterator
+changes the slowest, so the generated values are in lexicographic order. Example:
+
+```jldoctest
+julia> vec(collect(lexproduct(1:2,3:5)))
+6-element Array{Tuple{Int64,Int64},1}:
+ (1,3)
+ (1,4)
+ (1,5)
+ (2,3)
+ (2,4)
+ (2,5)
+```
+"""
+lexproduct(a, b) = LexProd2(a, b)
+
+lexproduct(a) = Prod1(a)
+
+eltype{I1,I2}(::Type{LexProd2{I1,I2}}) = Tuple{eltype(I1), eltype(I2)}
+
+iteratoreltype{I1,I2}(::Type{LexProd2{I1,I2}}) = and_iteratoreltype(iteratoreltype(I1),iteratoreltype(I2))
+iteratorsize{I1,I2}(::Type{LexProd2{I1,I2}}) = prod_iteratorsize(iteratorsize(I1),iteratorsize(I2))
+
+@inline function lex_prod_next(p, st)
+    s1, s2 = st[1], st[2]
+    v2, s2 = next(p.b, s2)
+
+    nv1 = st[3]
+    if isnull(nv1)
+        v1, s1 = next(p.a, s1)
+    else
+        v1 = nv1.value
+    end
+
+    if done(p.b, s2)
+        return (v1,v2), (s1, start(p.b), oftype(nv1,nothing), done(p.a,s1))
+    end
+    return (v1,v2), (s1, s2, Nullable(v1), false)
+end
+
+@inline next(p::LexProd2, st) = lex_prod_next(p, st)
+
+# n iterators
+immutable LexProd{I1, I2<:AbstractProdIterator} <: AbstractProdIterator
+    a::I1
+    b::I2
+end
+lexproduct(a, b, c...) = LexProd(a, lexproduct(b, c...))
+
+eltype{I1,I2}(::Type{LexProd{I1,I2}}) = tuple_type_cons(eltype(I1), eltype(I2))
+
+iteratoreltype{I1,I2}(::Type{LexProd{I1,I2}}) = and_iteratoreltype(iteratoreltype(I1),iteratoreltype(I2))
+iteratorsize{I1,I2}(::Type{LexProd{I1,I2}}) = prod_iteratorsize(iteratorsize(I1),iteratorsize(I2))
+
+@inline function next{I1,I2}(p::LexProd{I1,I2}, st)
+    x = lex_prod_next(p, st)
+    ((x[1][1],x[1][2]...), x[2])
+end
 
 
 # flatten an iterator of iterators
@@ -730,3 +775,5 @@ function next(itr::PartitionIterator, state)
     end
     return resize!(v, i), state
 end
+
+end

base/multidimensional.jl

@@ -381,7 +381,7 @@ end
 # and ensure the value to set is either an AbstractArray or a Repeated scalar
 # before redispatching to the _unsafe_batchsetindex!
 _iterable(v::AbstractArray) = v
-_iterable(v) = repeated(v)
+_iterable(v) = IterTools.repeated(v)
 @inline function _setindex!{T,N}(l::LinearIndexing, A::AbstractArray{T,N}, x, J::Vararg{Union{Real,AbstractArray,Colon},N})
     @boundscheck checkbounds(A, J...)
     _unsafe_setindex!(l, A, x, J...)

base/pkg/resolve.jl

@@ -66,7 +66,7 @@ function sanity_check(deps::Dict{String,Dict{VersionNumber,Available}},
 
     vers = Array{Tuple{String,VersionNumber,VersionNumber}}(0)
     for (p,d) in deps, vn in keys(d)
-        lvns = VersionNumber[filter(vn2->(vn2>vn), keys(d))...]
+        lvns = VersionNumber[IterTools.filter(vn2->(vn2>vn), keys(d))...]
         nvn = isempty(lvns) ? typemax(VersionNumber) : minimum(lvns)
         push!(vers, (p,vn,nvn))
     end