Massive Singles flatMap or Observers concatMap will crash
DaiyiLL opened this issue · comments
Massive Single flatMap or Observers concatMap will crash
crash:
EXC_BAD_ACCESS (code=2, address=0x70000ebacfe8)
// crash code，inner RxSwift
final private class FlatMap<SourceElement, SourceSequence: ObservableConvertibleType>: Producer<SourceSequence.Element> {
typealias Selector = (SourceElement) throws -> SourceSequence
private let source: Observable<SourceElement>
private let selector: Selector
init(source: Observable<SourceElement>, selector: @escaping Selector) {
self.source = source
self.selector = selector
}
override func run<Observer: ObserverType>(_ observer: Observer, cancel: Cancelable) -> (sink: Disposable, subscription: Disposable) where Observer.Element == SourceSequence.Element {
** //================ crash here ====================================**
let sink = FlatMapSink(selector: self.selector, observer: observer, cancel: cancel)
let subscription = sink.run(self.source)
return (sink: sink, subscription: subscription)
}
}// Single - extention
import RxSwift
import RxRelay
extension PrimitiveSequenceType where Trait == SingleTrait {
static func sequence<Collection: Swift.Collection>(_ list: Collection, maxCount: Int = 1) -> PrimitiveSequence<Trait, [Element]> where Collection.Element == PrimitiveSequence<Trait, Element> {
if list.count <= 0 {
return Single.just([])
}
var singles: [[PrimitiveSequence<Trait, Element>]] = []
list.forEach { element in
if var last = singles.last, last.count < maxCount {
last.append(element)
singles[singles.count - 1] = last
} else {
// 新建一个
let last = [element]
singles.append(last)
}
}
guard let firstSingles = singles.first else {
return Single.just([])
}
// var single: Observable<[Element]> = Single.zip(firstSingles).asObservable()
// for item in singles[1..<singles.count] {
// single = single.concatMap({ list in
// return Single.zip(item).map { newList in
// return list + newList
// }.asObservable()
// })
// }
// return single.asSingle()
guard let firstSingles = singles.first else {
return Single.just([])
}
var single: PrimitiveSequence<Trait, [Element]> = Single.zip(firstSingles)
for item in singles[1..<singles.count] {
single = single.flatMap { oldList in
return Single.zip(item).map { newList in
return oldList + newList
}
}
}
return single
}
}
// The test code
let singles: [Single<String>] = (0..<10000).compactMap { i in
return Single.just("\(i)")
}
let queue = DispatchQueue(label: "imageoperation")
let scheduler = SerialDispatchQueueScheduler(queue: queue, internalSerialQueueName: "1234")
Single.sequence(singles, maxCount: 4)
// Single.zip(singles)
.subscribe(on: scheduler)
.observe(on: MainScheduler.instance)
.subscribe(onSuccess: { [weak self] value in
print("result = \(value)")
print(Thread.current)
}, onFailure: { [weak self] error in
print(error)
}).disposed(by: self.disposeBag)
RxSwift/RxCocoa/RxBlocking/RxTest version/commit
6.5.0
Platform/Environment
iOS
How easy is to reproduce? (chances of successful reproduce after running the self contained code)
easy, 100% repro
Xcode version:
14.2
The code you posted doesn't compile. Please post a compilable and minimal example.
Now, I remove some code, try again. if not, I will give a example project.
Just cut the above code and paste it into Xcode. It doesn't compile. Fix the compile errors.
Cannot find 'FlatMapSink' in scope
Cannot find 'self' in scope; did you mean to use it in a type or extension context?
Cannot find type 'Producer' in scope
Expected expression
Expressions are not allowed at the top level
Method does not override any method from its superclass
Unary operator cannot be separated from its operand
I don't need a whole example project. Just compilable code that exhibits the error.
Note, this code compiles and works as expected:
let queue = DispatchQueue(label: "imageoperation")
let scheduler = SerialDispatchQueueScheduler(queue: queue, internalSerialQueueName: "1234")
func example() {
_ = Single.zip((0..<40000).map { Single.just("\($0)") })
.subscribe(on: scheduler)
.observe(on: MainScheduler.instance)
.subscribe(
onSuccess: { value in
print("result = \(value)")
print(Thread.current)
},
onFailure: { error in
print(error)
}
)
}
Please use flatMap. for example I will upload 200 jpeg images to server, zip(4) uploaded and upload the next zip(4) all the time, not zip(200) to server
var single: Single<String> = Single.just("-1")
(0..<10000).forEach { i in
single = single.flatMap({ str in
return Single.just("\(i)")
})
}
single.subscribe(on: scheduler)
.observe(on: MainScheduler.instance)
.subscribe(onSuccess: { [weak self] value in
print("result = \(value)")
print(Thread.current)
}, onFailure: { [weak self] error in
print(error)
}).disposed(by: self.disposeBag)This is a literal stack overflow. You get the same problem any time you try to nest 1500 or more calls deep in a dispatch queue no matter what the context. This is not an RxSwift problem.
There's a better way to solve this:
func uploadJpeg(data: Data) -> Single<String> {
Single.just(String(data: data, encoding: .utf8)!)
.delay(.milliseconds(100), scheduler: SerialDispatchQueueScheduler(qos: .background))
}
func example() {
let jpegs = (0..<10000).map { "\($0)".data(using: .utf8)! }
_ = Single.zip(jpegs.map { uploadJpeg(data: $0) })
.observe(on: MainScheduler.instance)
.subscribe(onSuccess: { value in
print("result = \(value)")
print(Thread.current)
}, onFailure: { error in
print(error)
})
}
The above simulates uploading 10000 images in parallel. (Note they won't actually go in parallel, the underlying URLSession object will only allow about 7 uploads at a time depending on hardware.) You will be notified when all the uploads are complete or one of them errors.
Understand, "1500 or more calls deep in a dispatch queue". Or I use DispatchSemaphore reduce call deeps, and thanks very much.
If you really want to break it up into four streams and don't want URLSession to take care of that, then it's a bit more complex, you can use the groupBy(keySelector:) operator.
Like this. Comments in the code to explain each line above the comment.
struct DataGroup: Hashable {
let group: Int
let data: Data
}
func uploadJpeg(data: Data) -> Single<String> {
Single.just(String(data: data, encoding: .utf8)!)
.delay(.milliseconds(10), scheduler: SerialDispatchQueueScheduler(qos: .background))
}
func example() {
let jpegs = (0..<10000).map { "\($0)".data(using: .utf8)! }
_ = Observable.merge(jpegs.enumerated().map { Observable.just(DataGroup(group: $0.offset % 4, data: $0.element)) })
/// wrap each jpeg into a DataGroup with it's group number, then merge them together, producing an `Observable<DataGroup>`
.groupBy(keySelector: { $0.group })
/// group the output of the previous line by their group numbers, producing an `Observable<GroupedObservable<Int, DataGroup>>`
.flatMap { $0.concatMap { uploadJpeg(data: $0.data) } }
/// setup each GroupedObservable by concatinating their uploads (one won't start until the previous one completes)
/// and flatMap the four GroupedObservables back together, producing an `Observable<Single<String>>`
.toArray()
/// wait until they all complete and gather up the results into an array, producing a `Single<[String]>`
.subscribe(onSuccess: { value in
print("result = \(value)")
print(Thread.current)
}, onFailure: { error in
print(error)
})
}
OK
Thanks Daniel, as always, for your smart feedback here :) I agree that this isn't an RxSwift issue.
I suggest switching the discussion to our Slack channel if any more needs arise here.
Thanks!