Needs["Streaming`"] ClearAll[lazyWeakCompositions] lazyWeakCompositions[ n_Integer?NonNegative, 0, chunkSize : _Integer?Positive : 10^5, opts : OptionsPattern[] ] := LazyListCreate[{}, chunkSize, opts] lazyWeakCompositions[ n_Integer?NonNegative, 1, chunkSize : _Integer?Positive : 10^5, opts : OptionsPattern[] ] := LazyListCreate[{{n}}, chunkSize, opts] lazyWeakCompositions[ n_Integer?NonNegative, k_Integer /; k >= 2, chunkSize : _Integer?Positive : 10^5, opts : OptionsPattern[] ] := With[{length = Binomial[n + k - 1, n]}, Module[ { active = False, compositionsLeft = length, prevComposition = Join[ConstantArray[0, k - 2], {-1, n + 1}] // Developer`ToPackedArray, lastNonZeroIndex = {k - 1} // Developer`ToPackedArray } , LazyListCreate[ IteratorCreate[ ListIterator, (active = True) &, Module[With[{realChunkSize = Min[chunkSize, compositionsLeft]}, If[realChunkSize === 0, {} ,(* else *) compositionsLeft -= realChunkSize; nextWeakCompositionsChunk[ n, k, prevComposition, lastNonZeroIndex, realChunkSize ] ] ] &, TrueQ[active] &, Remove[ active, compositionsLeft, prevComposition, lastNonZeroIndex ] & ], chunkSize, opts, "Length" -> length, "FiniteList" -> True ] ] ]
All compositions generated at once:
ClearAll[scanWeakCompositionsPermPartNonJoined] scanWeakCompositionsPermPartNonJoined[n_, k_] := ( Scan[IdentityLazyListBlock@Scan[Identity, weakCompositionsPermPartNonJoined[n, k], {2}]; DeleteFile@FileNames["*.mx", $StreamingCacheBase] )]
Compositions generated by permuting lazy partitions. Partitions are taken one by one, so in single chunk we have all permutations of given partition.
ClearAll[lazyWeakCompositionsPermPart, scanLazyWeakCompositionsPermPart] lazyWeakCompositionsPermPart[n_, k_] := Permutations /@ lazyIntegerPartitions[n, {k}, Range[0, n], 1] scanLazyWeakCompositionsPermPart[n_, k_] := ( Scan[Scan[Identity]LazyListBlock@Scan[Scan[Identity], lazyWeakCompositionsPermPart[n, k]]; DeleteFile@FileNames["*.mx", $StreamingCacheBase] )k]]
Compositions generated using our lazyWeakCompositions function in chunks of default length: 10^5.
ClearAll[scanLazyWeakCompositions] scanLazyWeakCompositions[n_, k_]k_, chunkSize_:= (10^5] := Scan[IdentityLazyListBlock@Scan[Identity, lazyWeakCompositions[n, k]]; DeleteFile@FileNames["*.mx"k, $StreamingCacheBase] )chunkSize]]
Since lazy lists cache their values and benchmark measures running time multiple times, we clear cache after each usage, using LazyListBlock (as per comment by Leonid).
$functionsList = {scanWeakCompositionsPermPartNonJoined, scanLazyWeakCompositionsPermPart, scanLazyWeakCompositions}; timeAndMemoryBenchmarkPlots[fixedKFunctionsList[8], Identity, {1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 14, 16, 18, 21, 24, 28, 32, 37, 42}, TimeConstraint -> Infinity, MemoryConstraint -> 2^31] timeAndMemoryBenchmarkPlots[fixedKFunctionsList[16], Identity, Range[16], TimeConstraint -> Infinity, MemoryConstraint -> 2^31] timeAndMemoryBenchmarkPlots[fixedKFunctionsList[32], Identity, Range[8], TimeConstraint -> Infinity, MemoryConstraint -> 2^31]


For k=8 number of permutations per partition is relatively small, so scanLazyWeakCompositionsPermPart uses a lot of relatively small chunks, in tested range of n. That's why it's considerably slower and it's memory usage increases slowly with n.
ClearAll[nextWeakCompositionsChunk] nextWeakCompositionsChunk = Compile[{{n, _Integer}, {k, _Integer}, {prevComposition, _Integer, 1}, {chunkSize, _Integer}}, Module[{lastNonZeroPosition = k, composition = prevComposition}, While[composition[[lastNonZeroPosition]] == 0, --lastNonZeroPosition]; Table[ ++composition[[lastNonZeroPosition - 1]]; If[lastNonZeroPosition == k, --composition[[-1]] ,(* else *) composition[[-1]] = composition[[lastNonZeroPosition]] - 1; composition[[lastNonZeroPosition]] = 0 ]; If[composition[[-1]] == 0, --lastNonZeroPosition ,(* else *) lastNonZeroPosition = k ]; composition , chunkSize ] ], RuntimeOptions -> "Speed", CompilationTarget -> "WVM" ]; ClearAll[weakCompositions] weakCompositions[n_Integer?NonNegative, 0] := {} weakCompositions[n_Integer?NonNegative, 1] := {{n}} weakCompositions[n_Integer?NonNegative, k_Integer /; k >= 2] := nextWeakCompositionsChunk[n, k, Join[ConstantArray[0, k - 2], {-1, n + 1}], Binomial[n + k - 1, n]] Needs["Streaming`"] ClearAll[lazyWeakCompositions] lazyWeakCompositions[n_Integer?NonNegative, 0, chunkSize : _Integer?Positive : 10^5, opts : OptionsPattern[]] := LazyListCreate[{}, chunkSize, opts] lazyWeakCompositions[n_Integer?NonNegative, 1, chunkSize : _Integer?Positive : 10^5, opts : OptionsPattern[]] := LazyListCreate[{{n}}, chunkSize, opts] lazyWeakCompositions[n_Integer?NonNegative, k_Integer /; k >= 2, chunkSize : _Integer?Positive : 10^5, opts : OptionsPattern[]] := With[{length = Binomial[n + k - 1, n]}, Module[{active = False, compositionsLeft = length, prevComposition = Join[ConstantArray[0, k - 2], {-1, n + 1}]}, LazyListCreate[ IteratorCreate[ ListIterator, (active = True) &, Module[With[{realChunkSize = Min[chunkSize, compositionsLeft], taken}, If[realChunkSize === 0, {} ,(* else **else*) compositionsLeft -= realChunkSize; With[{taken = nextWeakCompositionsChunk[n, k, prevComposition, realChunkSize];realChunkSize]}, prevComposition = Last[taken]; taken ] ] ] &, TrueQ[active] &, Remove[active, compositionsLeft, prevComposition] & ], chunkSize, opts, "Length" -> length, "FiniteList" -> True ] ] ]