cppcolors | Embarcadero RAD Studio Delphi C++Builder Blogs

Author: h.mohri

Convert from CSV file to “JSON file” of “TFDMemTable format”.

This program uses C++Builder 10.2 Tokyo Release 1.

include

<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><iostream></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><fstream></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><string></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><codecvt></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><FireDAC.Comp.Client.hpp></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><FireDAC.Stan.StorageJSON.hpp></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><FireDAC.Stan.Intf.hpp></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><memory></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><vector></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><sstream></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><mutex></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><thread></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><chrono></span>

#include <iostream>

#include <fstream>

#include <string>

#include <codecvt>

#include <FireDAC.Comp.Client.hpp>

#include <FireDAC.Stan.StorageJSON.hpp>

#include <FireDAC.Stan.Intf.hpp>

#include <memory>

#include <vector>

#include <sstream>

#include <mutex>

#include <thread>

#include <chrono>

code

<span class="synType" style="color:#3ec63e;">struct</span> _Tcsv_to_memtable
{
    std::mutex mx_;
    std::vector<std::thread> vth_;
    std::unique_ptr<TFDMemTable> mem_table_{std::make_unique<TFDMemTable>(<span class="synConstant" style="color:#ff6666;">nullptr</span>)};
    std::unique_ptr<TFDStanStorageJSONLink> jl_{std::make_unique<TFDStanStorageJSONLink>(<span class="synConstant" style="color:#ff6666;">nullptr</span>)};
    <span class="synType" style="color:#3ec63e;">void</span> create_column(std::vector<String>& v)
    {
        <span class="synComment" style="color:#4f80e5;">//Field creation of "TFDMemTable"</span>
        <span class="synComment" style="color:#4f80e5;">//The field name is the first line of CSV File</span>
        <span class="synStatement" style="color:#d88a17;">for</span> (<span class="synType" style="color:#3ec63e;">auto</span> s: v)
        {
            <span class="synType" style="color:#3ec63e;">auto</span> fld = mem_table_->FieldDefs->AddFieldDef();
            <span class="synComment" style="color:#4f80e5;">//Discard the "BOM (Byte Order Mark)" if present.</span>
            fld->Name = StringReplace(s, <span class="synSpecial" style="color:#c000c0;">L'x0feff'</span>, <span class="synConstant" style="color:#ff6666;">""</span>, TReplaceFlags());
            fld->DataType = TFieldType::ftWideString;
        }
        mem_table_->Active = <span class="synConstant" style="color:#ff6666;">true</span>;
    }
    <span class="synType" style="color:#3ec63e;">void</span> create_line(std::vector<String>& v)
    {
        <span class="synComment" style="color:#4f80e5;">//Add 1 line of "CSV File" to "TFDMemTable"</span>
        <span class="synType" style="color:#3ec63e;">int</span> i{<span class="synConstant" style="color:#ff6666;">0</span>};
        mem_table_->Append();
        <span class="synStatement" style="color:#d88a17;">for</span> (<span class="synType" style="color:#3ec63e;">auto</span> s: v)
        {
            <span class="synComment" style="color:#4f80e5;">//One item for one field.</span>
            mem_table_->Fields-><span class="synStatement" style="color:#d88a17;">operator</span>[](i)->AsWideString = s;
            ++i;
        }
        mem_table_->Post();
    }
    <span class="synType" style="color:#3ec63e;">void</span> create_line(std::wstring ws, <span class="synType" style="color:#3ec63e;">int</span> line_num)
    {
        std::lock_guard<std::mutex> lk(mx_);
        <span class="synStatement" style="color:#d88a17;">if</span> (ws.length() > <span class="synConstant" style="color:#ff6666;">0</span>)
        {
            <span class="synComment" style="color:#4f80e5;">//Inserted by dividing one line of CSV File to vector<String>.</span>
            std::vector<String> v1;
            std::wstring column_data;
            std::wistringstream line_data(ws);
            <span class="synStatement" style="color:#d88a17;">while</span> (std::getline(line_data, column_data, <span class="synConstant" style="color:#ff6666;">L','</span>))
            {
                v1.push_back(column_data.c_str());
            }
            <span class="synStatement" style="color:#d88a17;">switch</span> (line_num)
            {
            <span class="synStatement" style="color:#d88a17;">case</span> <span class="synConstant" style="color:#ff6666;">0</span>:
                create_column(v1);
                <span class="synStatement" style="color:#d88a17;">break</span>;
            <span class="synStatement" style="color:#d88a17;">default</span>:
                create_line(v1);
            }
        }

    }
    <span class="synType" style="color:#3ec63e;">void</span> threads_join()
    {
        <span class="synComment" style="color:#4f80e5;">//Wait for the termination of all threads.</span>
        <span class="synStatement" style="color:#d88a17;">for</span> (<span class="synType" style="color:#3ec63e;">auto</span>& th: vth_)
        {
            th.join();
        }
    }

};
<span class="synType" style="color:#3ec63e;">int</span> _tmain(<span class="synType" style="color:#3ec63e;">int</span> argc, _TCHAR* argv[])
{
    std::chrono::system_clock::time_point now = std::chrono::system_clock::now();
    _Tcsv_to_memtable memtable_;
    UnicodeString argv_     = argv[<span class="synConstant" style="color: #ff6666;">1</span>];  <span class="synComment" style="color:#4f80e5;">//Specify CSV Fire for input.</span>
    UnicodeString arg_o_    = argv[<span class="synConstant" style="color: #ff6666;">2</span>];  <span class="synComment" style="color:#4f80e5;">//JSON output destination specification made with "TFDMemTable".</span>
    <span class="synStatement" style="color:#d88a17;">if</span> (FileExists(argv_))
    {
        std::wfstream file_;
        file_.open(argv_.w_str(), std::ios::in);
        <span class="synComment" style="color:#4f80e5;">//The CSV file is utf8 format.</span>
        file_.imbue(std::locale(std::locale::empty(), <span class="synStatement" style="color:#d88a17;">new</span> std::codecvt_utf8<<span class="synType" style="color:#3ec63e;">wchar_t</span>>));

        std::wstring buff;
        <span class="synType" style="color:#3ec63e;">int</span> line_num_{<span class="synConstant" style="color:#ff6666;">0</span>};
        <span class="synStatement" style="color:#d88a17;">while</span> (std::getline(file_, buff))
        {
            <span class="synComment" style="color:#4f80e5;">//The append processing to "TFDMemTable" threads line by line.</span>
            memtable_.vth_.push_back(std::thread([&memtable_, &buff, line_num_](){
                memtable_.create_line(buff, line_num_);
                }));
            <span class="synComment" style="color:#4f80e5;">//Since the command line is multibyte, AnsiString conversion is done.</span>
            AnsiString as_ = buff.c_str();
            std::cout << as_ << std::endl;
            ++line_num_;
        }

        memtable_.threads_join();
        <span class="synComment" style="color:#4f80e5;">//Finally, save the TFDMemTable contents as a JSON file.</span>
        memtable_.mem_table_->SaveToFile(arg_o_, TFDStorageFormat::sfJSON);

    }
    <span class="synType" style="color:#3ec63e;">auto</span> keika =  std::chrono::system_clock::now() - now;
    std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(keika).count();
    std::cout << <span class="synConstant" style="color:#ff6666;">" millisecond."</span> << std::endl ;
    <span class="synStatement" style="color:#d88a17;">return</span> <span class="synConstant" style="color:#ff6666;">0</span>;
}

100

101

102

103

struct _Tcsv_to_memtable

{

std::mutex mx_;

std::vector<std::thread> vth_;

std::unique_ptr<TFDMemTable> mem_table_{std::make_unique<TFDMemTable>(nullptr)};

std::unique_ptr<TFDStanStorageJSONLink> jl_{std::make_unique<TFDStanStorageJSONLink>(nullptr)};

void create_column(std::vector<String>& v)

{

//Field creation of "TFDMemTable"

//The field name is the first line of CSV File

for (auto s: v)

{

auto fld = mem_table_->FieldDefs->AddFieldDef();

//Discard the "BOM (Byte Order Mark)" if present.

fld->Name = StringReplace(s, L'x0feff', "", TReplaceFlags());

fld->DataType = TFieldType::ftWideString;

}

mem_table_->Active = true;

}

void create_line(std::vector<String>& v)

{

//Add 1 line of "CSV File" to "TFDMemTable"

int i{0};

mem_table_->Append();

for (auto s: v)

{

//One item for one field.

mem_table_->Fields->operator[](i)->AsWideString = s;

++i;

}

mem_table_->Post();

}

void create_line(std::wstring ws, int line_num)

{

std::lock_guard<std::mutex> lk(mx_);

if (ws.length() > 0)

{

//Inserted by dividing one line of CSV File to vector<String>.

std::vector<String> v1;

std::wstring column_data;

std::wistringstream line_data(ws);

while (std::getline(line_data, column_data, L','))

{

v1.push_back(column_data.c_str());

}

switch (line_num)

{

case 0:

create_column(v1);

break;

default:

create_line(v1);

}

void threads_join()

{

//Wait for the termination of all threads.

for (auto& th: vth_)

{

th.join();

}

};

int _tmain(int argc, _TCHAR* argv[])

{

std::chrono::system_clock::time_point now = std::chrono::system_clock::now();

_Tcsv_to_memtable memtable_;

UnicodeString argv_ = argv[1]; //Specify CSV Fire for input.

UnicodeString arg_o_ = argv[2]; //JSON output destination specification made with "TFDMemTable".

if (FileExists(argv_))

{

std::wfstream file_;

file_.open(argv_.w_str(), std::ios::in);

//The CSV file is utf8 format.

file_.imbue(std::locale(std::locale::empty(), new std::codecvt_utf8<wchar_t>));

std::wstring buff;

int line_num_{0};

while (std::getline(file_, buff))

{

//The append processing to "TFDMemTable" threads line by line.

memtable_.vth_.push_back(std::thread([&memtable_, &buff, line_num_](){

memtable_.create_line(buff, line_num_);

}));

//Since the command line is multibyte, AnsiString conversion is done.

AnsiString as_ = buff.c_str();

std::cout << as_ << std::endl;

++line_num_;

}

memtable_.threads_join();

//Finally, save the TFDMemTable contents as a JSON file.

memtable_.mem_table_->SaveToFile(arg_o_, TFDStorageFormat::sfJSON);

}

auto keika = std::chrono::system_clock::now() - now;

std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(keika).count();

std::cout << " millisecond." << std::endl ;

return 0;

}

There is a project file in github, you can try it right away.
GitHub – mojeld/csvfstream_to_TFDMemTable: Create json file of TFDMemTable format from CSV file.

f:id:mojeld:20170901174729p:plain

Reduce development time and get to market faster with RAD Studio, Delphi, or C++Builder.
Design. Code. Compile. Deploy.
Start Free Trial Upgrade Today

Free Delphi Community Edition Free C++Builder Community Edition

Convert CSV file to TFDMemTable format JSON file.

Convert from CSV file to “JSON file” of “TFDMemTable format”.

include

code

Leave a ReplyCancel reply

Search

Something Fresh

WebStencils and HTMX: Free Guide to Fast Web Development

RAD Studio 12.2 Athens Inline Patch 1 Available

The Untapped Potential of AI in Desktop Applications: “Power-Charging” Our Delphi Community

Popular Posts

RAD Studio 12.2 Athens Inline Patch 1 Available

Announcing the Availability of RAD Studio 12.2 Athens

Delphi 12 And C++Builder 12 Community Editions Released!

InterBase ODBC Driver on GitHub

Embarcadero Partners with Raize Software for KSVC Maintenance

Categories

Unknown Feed

Unknown Feed

Categories

Useful Links

Follow us

Convert CSV file to TFDMemTable format JSON file.

Convert from CSV file to “JSON file” of “TFDMemTable format”.

include

code

Leave a ReplyCancel reply

Join Our Global Developer Community

Search

Something Fresh

Popular Posts

Categories

Categories

Useful Links

Follow us