4

I am dealing big libpacp files with the program below.

I am confused about the real max size of memory that stringstream can allocate from the OS.

The first part of code is the program for processing libpacp files.

The second part is the program for testing.

Environment: Windows 10, VS, complied in Win32-Released(32-bit) mode.

First Part:

#include <fstream> #include <iostream> #include <sstream> #include <string> #include <ctime> #include <cstdio> #define HeaderBytes 24 #define MaxPkgBytes 65544 //65536+8 #define KeepDays 7 #define KeepSeconds (KeepDays*86400) #define StartTimeOffset (-1*86400) // -1 day using namespace std; typedef struct{ int size; char data[MaxPkgBytes]; }pkg; int catoi(const char* ca){ char tmp[4]; int* iptr; for (int i = 0; i < 4; i++){ tmp[i] = ca[3 - i]; } iptr = reinterpret_cast<int*>(tmp); return *iptr; } #ifdef _MSC_VER #include <windows.h> #include <iomanip> wstring str2wstr(const std::string& s) { int len; int slength = (int)s.length() + 1; len = MultiByteToWideChar(CP_ACP, 0, s.c_str(), slength, 0, 0); wchar_t* buf = new wchar_t[len]; MultiByteToWideChar(CP_ACP, 0, s.c_str(), slength, buf, len); wstring wstr(buf); return wstr; } #endif // _MSC_VER int main(int argc, char** argv){ string inFileName, outFileName; stringstream outBuf; fstream fs_in, fs_out; char buf_char; int buf_int, headercount = 0, curPkgIdx= 0, lastPkgIdx = 1, tmp; bool isBroken = false, isValid; clock_t mytime; unsigned int StartTime = 0, PkgTime; pkg buf_pkg[2]; if (argc != 2){ return 1; } inFileName = argv[1]; fs_in.open(inFileName, ios::binary | ios::in); if (!fs_in){ cout << "Can't open the file: " << inFileName << endl; return 1; } outFileName = inFileName; outFileName.insert(outFileName.rfind('.'), "_integrated"); fs_out.open(outFileName, ios::binary | ios::out); if (!fs_out){ cout << "Can't open the file: " << outFileName << endl; return 1; } int invalidPConuter = 0; long long outBufMaxPos = 0; buf_pkg[0].size = 0; buf_pkg[1].size = 0; mytime = clock(); fs_in.read(buf_pkg[curPkgIdx].data, HeaderBytes); outBuf.write(buf_pkg[curPkgIdx].data, HeaderBytes); if (fs_in){ fs_in.read(buf_pkg[curPkgIdx].data, 4); StartTime = catoi(buf_pkg[curPkgIdx].data); StartTime += StartTimeOffset; fs_in.seekg(-4, ios_base::cur); } cout << "start" << endl; while (fs_in.get(buf_char)){ fs_in.seekg(-1, ios_base::cur); if (buf_char == -95 ){ //0xa1 fs_in.read(reinterpret_cast<char*>(&buf_int), sizeof(int)); if (buf_int == 0xd4c3b2a1){ //a1b2 c3d4 fs_in.seekg(HeaderBytes-4, ios_base::cur); headercount++; } else fs_in.seekg(-4, ios_base::cur); } else{ fs_in.read(buf_pkg[curPkgIdx].data, 16); PkgTime = catoi(buf_pkg[curPkgIdx].data); /*Set isValid*/ if (PkgTime - StartTime < KeepSeconds) isValid = true; else isValid = false; if (isValid){ //last packetage is valid /*store size of packetage*/ buf_pkg[curPkgIdx].size = catoi(buf_pkg[curPkgIdx].data + 8); /*store size of packetage*/ if (buf_pkg[curPkgIdx].size > MaxPkgBytes) isValid = false; } if (isValid) //Pass packet size check { /*read packetage data*/ fs_in.read(buf_pkg[curPkgIdx].data + 16, buf_pkg[curPkgIdx].size); buf_pkg[curPkgIdx].size += 16; /*read packetage data*/ /*write last packetage data*/ outBuf.write(buf_pkg[lastPkgIdx].data, buf_pkg[lastPkgIdx].size); if (static_cast<long long>(outBuf.tellp()) > outBufMaxPos) { outBufMaxPos = static_cast<long long>(outBuf.tellp()); } else if (static_cast<long long>(outBuf.tellp()) == -1) { cout << "outBufMaxPos: " << outBufMaxPos << endl; system("pause"); } if (outBuf.tellp() >= 0x40000000 - MaxPkgBytes) // 1GB { cout << "write" << endl; fs_out << outBuf.rdbuf(); outBuf.str(""); outBuf.clear(); } /*write last packetage data*/ /*swap idx of buffer*/ tmp = curPkgIdx; curPkgIdx = lastPkgIdx; lastPkgIdx = tmp; /*swap idx of buffer*/ } if (!isValid) { ++invalidPConuter; isBroken = true; fs_in.seekg(-buf_pkg[lastPkgIdx].size - 15, ios_base::cur); /*search correct packetage byte by byte*/ int tmpflag = 0; /*Let PkgTime be invalid. If packet is invalid because of its size, original PkgTime was valid*/ PkgTime = StartTime + KeepSeconds; while (PkgTime - StartTime >= KeepSeconds && fs_in.read(buf_pkg[curPkgIdx].data, 4)){ PkgTime = catoi(buf_pkg[curPkgIdx].data); fs_in.seekg(-3, ios_base::cur); } fs_in.seekg(-1, ios_base::cur); /*search correct packetage byte by byte*/ buf_pkg[lastPkgIdx].size = 0; //reset the size of the invalid packetage } } } fs_in.close(); mytime = clock() - mytime; cout << "Repair pacp: " << mytime << " miniseconds." << endl; cout << "Number of deleted headers: " << headercount << endl; mytime = clock(); if (headercount || isBroken){ fs_out << outBuf.rdbuf(); fs_out.close(); #ifdef _MSC_VER wstring originFileName, newFileName; originFileName = str2wstr(inFileName); newFileName = str2wstr(inFileName.insert(inFileName.rfind("."), "_origin")); int flag = MoveFileExW(originFileName.c_str(), newFileName.c_str(), 0); if (!flag) { cout << "fail to rename origin file" << endl; cout << showbase // show the 0x prefix << internal // fill between the prefix and the number << setfill('0'); // fill with 0s cout << "Error code: " << hex << setw(4) << GetLastError() << dec << endl; } else { newFileName = originFileName; originFileName = str2wstr(outFileName); flag = MoveFileExW(originFileName.c_str(), newFileName.c_str(), 0); if (!flag) { cout << "fail to rename output file" << endl; cout << showbase // show the 0x prefix << internal // fill between the prefix and the number << setfill('0'); // fill with 0s cout << "Error code: " << hex << setw(4) << GetLastError() << dec << endl; } } #endif //_MSC_VER } else { wstring tmpwstr = str2wstr(outFileName); fs_out.close(); if (!DeleteFileW(tmpwstr.c_str())) { cout << "Cannot deleted tmp file (integrated)" << endl; } cout << "The file is completed. Do nothing." << endl; } mytime = clock() - mytime; cout << "Rename file: " << mytime << " miniseconds." << endl; system("pause"); return 0; }

Pseudo code of the first part:

using namespace std; int main(int argc, char** argv){ //leave over the varibles string inFileName, outFileName; fstream fs_out; char buf_char; int buf_int, headercount = 0, curPkgIdx= 0, lastPkgIdx = 1, tmp; bool isBroken = false, isValid; clock_t mytime; unsigned int StartTime = 0, PkgTime; pkg buf_pkg[2]; int invalidPConuter = 0; long long outBufMaxPos = 0; //the varibles will be mentioned fstream fs_in; stringstream outBuf; fs_in.read(Header); outBuf.write(Header); if (fs_in){ StartTime = first_packet_time + StartTimeOffset; } while (!fs_in.eof()){ if (a header read from fs_in){ skip the block of header } else{ fs_in.read(packet header); if (time of packet isValid){ check size of packet } if (size and time isValid) { fs_in.read(packet data); outBuf.write(packet data); if(outBuf out of range) { print(max stringstream size) system("pause"); } if (outBuf size >= 1GB) { write outBuf into output file } } if (size or time isNotValid) { find next valid packet byte by byte } } } fs_in.close(); system("pause"); return 0; }

Second Part:

#include <iostream> #include <typeinfo> #include <sstream> #include <string> using namespace std; #define testsize (80*1024*1024) int main() { stringstream ss; char* buf = new char[testsize]; int i = 0; memset(buf, 'a', testsize); while (i < 30) { ss.write(buf, testsize); cout << ss.tellp()/1024/1024 << endl; ++i; } system("pause"); }

In the first part, the stringstream's max size is limited to about 674MB.

But in the second part, stringstream's max size is limited to about 2GB.

Why they are different?

And how can I get real max size of stringstream without write anything into it?

I have searched related problem, but the answers didn't help me.

Improve this question
10
  • 2
    Why not switch to 64 bit and then you don't need to worry about it ? Commented Sep 20, 2016 at 15:44
  • 4
    Giving us the full code is giving us a hard time understanding what your question is. Can you please produce a short code specific to your problem? Commented Sep 20, 2016 at 15:44
  • 1
    Because the program may have to run on a x86 machine. Commented Sep 20, 2016 at 15:45
  • 1
    Why do you even need to know? What kind of problem warrants unusually large stringstream buffers? Commented Sep 20, 2016 at 15:45
  • On x86 you are limited not only by RAM but also by the 2GB address space. Commented Sep 20, 2016 at 15:46

3 Answers 3

Reset to default
3

The short answer is that you generally can't/won't know except by trying it.

The OS has a single pool of memory. That pool is shared between all processes currently executing on the system (plus a few things like device drivers that aren't exactly part of a process, but that distinction doesn't matter much for the moment).

In a typical case, the overall size of the pool isn't known and generally isn't knowable. It may change dynamically, as (for example) disks are added to/removed from the system.

The percentage of that pool that's available to any one process on the system generally isn't knowable either. Other processes are starting and stopping, which typically allocate and free memory, and many allocate and free memory during operation as well.

All of this is happening dynamically, so an attempt at allocation might succeed one moment, fail the next, and succeed again a few moments later. If the OS provided (for example) a function that would tell you how much memory was available when it was called, the result might easily be wrong before it was returned to the caller.

There are some hard limits. For an obvious one, a 32-bit process only has a 4 Gigabyte address space. Attempting to allocate (say) 8 gigabytes via normal methods (e.g., new) can't possibly succeed--ever.

A single allocation of size N needs a range of N contiguous bytes in that address range. Especially after a process has been running for a while, the available address space (independent of the underlying memory) will tend to get fragmented, so regardless of available memory, the largest single allocation that can succeed will be the size of the largest fragment of available address space.

In some cases there are also "soft" limits. For example, in Windows1 you can create a "job object", and specify the maximum memory used by processes running in that job object. This can prevent an allocation from succeeding even if physical RAM is available.

So, at any given moment, the maximum allocation that can succeed is the minimum of any of a half dozen (or so) different factors, nearly all of which are open to almost unpredictable changes. The only realistic way to know what will work is to try to allocate what you need, and see if that succeeds.

1. Here I use Windows as an example because the question refers to Windows. Although mechanisms and names vary, the basic idea is far from unique to Windows; most other OSes provide similar capabilities.

Improve this answer
Sign up to request clarification or add additional context in comments.

Comments

1

It seems that in MSVC-StandardLibrary, the size of the buffer is limited to INT_MAX = 2GB : https://github.com/microsoft/STL/blob/cf1313c39169dc376761eddee23c5e408e01aaa9/stl/inc/sstream#L252-L261

The issue has been reported several times: https://github.com/microsoft/STL/issues/388 https://github.com/microsoft/STL/issues/578 https://developercommunity.visualstudio.com/t/290124

but I don't think that they ever prioritized it for development because they think that a buffer is smth temporary that shouldn't supposed to be very big. In Gcc it isn't so, as the limits seem to be based on __string_type::size_type https://github.com/gcc-mirror/gcc/blob/master/libstdc%2B%2B-v3/include/std/sstream which is not so explicitly provided. So, it goes up to memory available.

Below are the results for Linux x64 platforms:

Improve this answer

Comments

0

The difference is likely that your first program has other stuff in its memory space besides your stringstream's buffer. As your program allocates and frees blocks of memory, the address space gets fragmented. You may have gigabytes of address space left, but there's no single, contiguous, gap anywhere that can hold a giant buffer. You also leak memory from str2wstr.

In your second program this isn't an issue because you only ever allocate two blocks of memory at a time. Your address space doesn't get fragmented, so your stringstream's buffer is able to use almost all of your available memory.

Improve this answer

Comments

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.