Author: h.mohri
Convert from CSV file to “JSON file” of “TFDMemTable format”.
This program uses C++Builder 10.2 Tokyo Release 1.
include
|
1 2 3 4 5 6 7 8 9 10 11 12 13 |
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><iostream></span> <span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><fstream></span> <span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><string></span> <span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><codecvt></span> <span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><FireDAC.Comp.Client.hpp></span> <span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><FireDAC.Stan.StorageJSON.hpp></span> <span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><FireDAC.Stan.Intf.hpp></span> <span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><memory></span> <span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><vector></span> <span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><sstream></span> <span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><mutex></span> <span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><thread></span> <span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><chrono></span> |
code
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 |
<span class="synType" style="color:#3ec63e;">struct</span> _Tcsv_to_memtable { std::mutex mx_; std::vector<std::thread> vth_; std::unique_ptr<TFDMemTable> mem_table_{std::make_unique<TFDMemTable>(<span class="synConstant" style="color:#ff6666;">nullptr</span>)}; std::unique_ptr<TFDStanStorageJSONLink> jl_{std::make_unique<TFDStanStorageJSONLink>(<span class="synConstant" style="color:#ff6666;">nullptr</span>)}; <span class="synType" style="color:#3ec63e;">void</span> create_column(std::vector<String>& v) { <span class="synComment" style="color:#4f80e5;">//Field creation of "TFDMemTable"</span> <span class="synComment" style="color:#4f80e5;">//The field name is the first line of CSV File</span> <span class="synStatement" style="color:#d88a17;">for</span> (<span class="synType" style="color:#3ec63e;">auto</span> s: v) { <span class="synType" style="color:#3ec63e;">auto</span> fld = mem_table_->FieldDefs->AddFieldDef(); <span class="synComment" style="color:#4f80e5;">//Discard the "BOM (Byte Order Mark)" if present.</span> fld->Name = StringReplace(s, <span class="synSpecial" style="color:#c000c0;">L'x0feff'</span>, <span class="synConstant" style="color:#ff6666;">""</span>, TReplaceFlags()); fld->DataType = TFieldType::ftWideString; } mem_table_->Active = <span class="synConstant" style="color:#ff6666;">true</span>; } <span class="synType" style="color:#3ec63e;">void</span> create_line(std::vector<String>& v) { <span class="synComment" style="color:#4f80e5;">//Add 1 line of "CSV File" to "TFDMemTable"</span> <span class="synType" style="color:#3ec63e;">int</span> i{<span class="synConstant" style="color:#ff6666;">0</span>}; mem_table_->Append(); <span class="synStatement" style="color:#d88a17;">for</span> (<span class="synType" style="color:#3ec63e;">auto</span> s: v) { <span class="synComment" style="color:#4f80e5;">//One item for one field.</span> mem_table_->Fields-><span class="synStatement" style="color:#d88a17;">operator</span>[](i)->AsWideString = s; ++i; } mem_table_->Post(); } <span class="synType" style="color:#3ec63e;">void</span> create_line(std::wstring ws, <span class="synType" style="color:#3ec63e;">int</span> line_num) { std::lock_guard<std::mutex> lk(mx_); <span class="synStatement" style="color:#d88a17;">if</span> (ws.length() > <span class="synConstant" style="color:#ff6666;">0</span>) { <span class="synComment" style="color:#4f80e5;">//Inserted by dividing one line of CSV File to vector<String>.</span> std::vector<String> v1; std::wstring column_data; std::wistringstream line_data(ws); <span class="synStatement" style="color:#d88a17;">while</span> (std::getline(line_data, column_data, <span class="synConstant" style="color:#ff6666;">L','</span>)) { v1.push_back(column_data.c_str()); } <span class="synStatement" style="color:#d88a17;">switch</span> (line_num) { <span class="synStatement" style="color:#d88a17;">case</span> <span class="synConstant" style="color:#ff6666;">0</span>: create_column(v1); <span class="synStatement" style="color:#d88a17;">break</span>; <span class="synStatement" style="color:#d88a17;">default</span>: create_line(v1); } } } <span class="synType" style="color:#3ec63e;">void</span> threads_join() { <span class="synComment" style="color:#4f80e5;">//Wait for the termination of all threads.</span> <span class="synStatement" style="color:#d88a17;">for</span> (<span class="synType" style="color:#3ec63e;">auto</span>& th: vth_) { th.join(); } } }; <span class="synType" style="color:#3ec63e;">int</span> _tmain(<span class="synType" style="color:#3ec63e;">int</span> argc, _TCHAR* argv[]) { std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); _Tcsv_to_memtable memtable_; UnicodeString argv_ = argv[<span class="synConstant" style="color: #ff6666;">1</span>]; <span class="synComment" style="color:#4f80e5;">//Specify CSV Fire for input.</span> UnicodeString arg_o_ = argv[<span class="synConstant" style="color: #ff6666;">2</span>]; <span class="synComment" style="color:#4f80e5;">//JSON output destination specification made with "TFDMemTable".</span> <span class="synStatement" style="color:#d88a17;">if</span> (FileExists(argv_)) { std::wfstream file_; file_.open(argv_.w_str(), std::ios::in); <span class="synComment" style="color:#4f80e5;">//The CSV file is utf8 format.</span> file_.imbue(std::locale(std::locale::empty(), <span class="synStatement" style="color:#d88a17;">new</span> std::codecvt_utf8<<span class="synType" style="color:#3ec63e;">wchar_t</span>>)); std::wstring buff; <span class="synType" style="color:#3ec63e;">int</span> line_num_{<span class="synConstant" style="color:#ff6666;">0</span>}; <span class="synStatement" style="color:#d88a17;">while</span> (std::getline(file_, buff)) { <span class="synComment" style="color:#4f80e5;">//The append processing to "TFDMemTable" threads line by line.</span> memtable_.vth_.push_back(std::thread([&memtable_, &buff, line_num_](){ memtable_.create_line(buff, line_num_); })); <span class="synComment" style="color:#4f80e5;">//Since the command line is multibyte, AnsiString conversion is done.</span> AnsiString as_ = buff.c_str(); std::cout << as_ << std::endl; ++line_num_; } memtable_.threads_join(); <span class="synComment" style="color:#4f80e5;">//Finally, save the TFDMemTable contents as a JSON file.</span> memtable_.mem_table_->SaveToFile(arg_o_, TFDStorageFormat::sfJSON); } <span class="synType" style="color:#3ec63e;">auto</span> keika = std::chrono::system_clock::now() - now; std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(keika).count(); std::cout << <span class="synConstant" style="color:#ff6666;">" millisecond."</span> << std::endl ; <span class="synStatement" style="color:#d88a17;">return</span> <span class="synConstant" style="color:#ff6666;">0</span>; } |
There is a project file in github, you can try it right away.
GitHub – mojeld/csvfstream_to_TFDMemTable: Create json file of TFDMemTable format from CSV file.
Reduce development time and get to market faster with RAD Studio, Delphi, or C++Builder.
Design. Code. Compile. Deploy.
Free Delphi Community Edition Free C++Builder Community Edition
