cppcolors | Embarcadero RAD Studio Delphi C++Builder Blogs

Author: h.mohri

Convert from CSV file to “JSON file” of “TFDMemTable format”.

This program uses C++Builder 10.2 Tokyo Release 1.

include

<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><iostream></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><fstream></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><string></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><codecvt></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><FireDAC.Comp.Client.hpp></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><FireDAC.Stan.StorageJSON.hpp></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><FireDAC.Stan.Intf.hpp></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><memory></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><vector></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><sstream></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><mutex></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><thread></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><chrono></span>

#include <iostream>

#include <fstream>

#include <string>

#include <codecvt>

#include <FireDAC.Comp.Client.hpp>

#include <FireDAC.Stan.StorageJSON.hpp>

#include <FireDAC.Stan.Intf.hpp>

#include <memory>

#include <vector>

#include <sstream>

#include <mutex>

#include <thread>

#include <chrono>

code

<span class="synType" style="color:#3ec63e;">struct</span> _Tcsv_to_memtable
{
    std::mutex mx_;
    std::vector<std::thread> vth_;
    std::unique_ptr<TFDMemTable> mem_table_{std::make_unique<TFDMemTable>(<span class="synConstant" style="color:#ff6666;">nullptr</span>)};
    std::unique_ptr<TFDStanStorageJSONLink> jl_{std::make_unique<TFDStanStorageJSONLink>(<span class="synConstant" style="color:#ff6666;">nullptr</span>)};
    <span class="synType" style="color:#3ec63e;">void</span> create_column(std::vector<String>& v)
    {
        <span class="synComment" style="color:#4f80e5;">//Field creation of "TFDMemTable"</span>
        <span class="synComment" style="color:#4f80e5;">//The field name is the first line of CSV File</span>
        <span class="synStatement" style="color:#d88a17;">for</span> (<span class="synType" style="color:#3ec63e;">auto</span> s: v)
        {
            <span class="synType" style="color:#3ec63e;">auto</span> fld = mem_table_->FieldDefs->AddFieldDef();
            <span class="synComment" style="color:#4f80e5;">//Discard the "BOM (Byte Order Mark)" if present.</span>
            fld->Name = StringReplace(s, <span class="synSpecial" style="color:#c000c0;">L'x0feff'</span>, <span class="synConstant" style="color:#ff6666;">""</span>, TReplaceFlags());
            fld->DataType = TFieldType::ftWideString;
        }
        mem_table_->Active = <span class="synConstant" style="color:#ff6666;">true</span>;
    }
    <span class="synType" style="color:#3ec63e;">void</span> create_line(std::vector<String>& v)
    {
        <span class="synComment" style="color:#4f80e5;">//Add 1 line of "CSV File" to "TFDMemTable"</span>
        <span class="synType" style="color:#3ec63e;">int</span> i{<span class="synConstant" style="color:#ff6666;">0</span>};
        mem_table_->Append();
        <span class="synStatement" style="color:#d88a17;">for</span> (<span class="synType" style="color:#3ec63e;">auto</span> s: v)
        {
            <span class="synComment" style="color:#4f80e5;">//One item for one field.</span>
            mem_table_->Fields-><span class="synStatement" style="color:#d88a17;">operator</span>[](i)->AsWideString = s;
            ++i;
        }
        mem_table_->Post();
    }
    <span class="synType" style="color:#3ec63e;">void</span> create_line(std::wstring ws, <span class="synType" style="color:#3ec63e;">int</span> line_num)
    {
        std::lock_guard<std::mutex> lk(mx_);
        <span class="synStatement" style="color:#d88a17;">if</span> (ws.length() > <span class="synConstant" style="color:#ff6666;">0</span>)
        {
            <span class="synComment" style="color:#4f80e5;">//Inserted by dividing one line of CSV File to vector<String>.</span>
            std::vector<String> v1;
            std::wstring column_data;
            std::wistringstream line_data(ws);
            <span class="synStatement" style="color:#d88a17;">while</span> (std::getline(line_data, column_data, <span class="synConstant" style="color:#ff6666;">L','</span>))
            {
                v1.push_back(column_data.c_str());
            }
            <span class="synStatement" style="color:#d88a17;">switch</span> (line_num)
            {
            <span class="synStatement" style="color:#d88a17;">case</span> <span class="synConstant" style="color:#ff6666;">0</span>:
                create_column(v1);
                <span class="synStatement" style="color:#d88a17;">break</span>;
            <span class="synStatement" style="color:#d88a17;">default</span>:
                create_line(v1);
            }
        }

    }
    <span class="synType" style="color:#3ec63e;">void</span> threads_join()
    {
        <span class="synComment" style="color:#4f80e5;">//Wait for the termination of all threads.</span>
        <span class="synStatement" style="color:#d88a17;">for</span> (<span class="synType" style="color:#3ec63e;">auto</span>& th: vth_)
        {
            th.join();
        }
    }

};
<span class="synType" style="color:#3ec63e;">int</span> _tmain(<span class="synType" style="color:#3ec63e;">int</span> argc, _TCHAR* argv[])
{
    std::chrono::system_clock::time_point now = std::chrono::system_clock::now();
    _Tcsv_to_memtable memtable_;
    UnicodeString argv_     = argv[<span class="synConstant" style="color: #ff6666;">1</span>];  <span class="synComment" style="color:#4f80e5;">//Specify CSV Fire for input.</span>
    UnicodeString arg_o_    = argv[<span class="synConstant" style="color: #ff6666;">2</span>];  <span class="synComment" style="color:#4f80e5;">//JSON output destination specification made with "TFDMemTable".</span>
    <span class="synStatement" style="color:#d88a17;">if</span> (FileExists(argv_))
    {
        std::wfstream file_;
        file_.open(argv_.w_str(), std::ios::in);
        <span class="synComment" style="color:#4f80e5;">//The CSV file is utf8 format.</span>
        file_.imbue(std::locale(std::locale::empty(), <span class="synStatement" style="color:#d88a17;">new</span> std::codecvt_utf8<<span class="synType" style="color:#3ec63e;">wchar_t</span>>));

        std::wstring buff;
        <span class="synType" style="color:#3ec63e;">int</span> line_num_{<span class="synConstant" style="color:#ff6666;">0</span>};
        <span class="synStatement" style="color:#d88a17;">while</span> (std::getline(file_, buff))
        {
            <span class="synComment" style="color:#4f80e5;">//The append processing to "TFDMemTable" threads line by line.</span>
            memtable_.vth_.push_back(std::thread([&memtable_, &buff, line_num_](){
                memtable_.create_line(buff, line_num_);
                }));
            <span class="synComment" style="color:#4f80e5;">//Since the command line is multibyte, AnsiString conversion is done.</span>
            AnsiString as_ = buff.c_str();
            std::cout << as_ << std::endl;
            ++line_num_;
        }

        memtable_.threads_join();
        <span class="synComment" style="color:#4f80e5;">//Finally, save the TFDMemTable contents as a JSON file.</span>
        memtable_.mem_table_->SaveToFile(arg_o_, TFDStorageFormat::sfJSON);

    }
    <span class="synType" style="color:#3ec63e;">auto</span> keika =  std::chrono::system_clock::now() - now;
    std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(keika).count();
    std::cout << <span class="synConstant" style="color:#ff6666;">" millisecond."</span> << std::endl ;
    <span class="synStatement" style="color:#d88a17;">return</span> <span class="synConstant" style="color:#ff6666;">0</span>;
}

100

101

102

103

struct _Tcsv_to_memtable

{

std::mutex mx_;

std::vector<std::thread> vth_;

std::unique_ptr<TFDMemTable> mem_table_{std::make_unique<TFDMemTable>(nullptr)};

std::unique_ptr<TFDStanStorageJSONLink> jl_{std::make_unique<TFDStanStorageJSONLink>(nullptr)};

void create_column(std::vector<String>& v)

{

//Field creation of "TFDMemTable"

//The field name is the first line of CSV File

for (auto s: v)

{

auto fld = mem_table_->FieldDefs->AddFieldDef();

//Discard the "BOM (Byte Order Mark)" if present.

fld->Name = StringReplace(s, L'x0feff', "", TReplaceFlags());

fld->DataType = TFieldType::ftWideString;

}

mem_table_->Active = true;

}

void create_line(std::vector<String>& v)

{

//Add 1 line of "CSV File" to "TFDMemTable"

int i{0};

mem_table_->Append();

for (auto s: v)

{

//One item for one field.

mem_table_->Fields->operator[](i)->AsWideString = s;

++i;

}

mem_table_->Post();

}

void create_line(std::wstring ws, int line_num)

{

std::lock_guard<std::mutex> lk(mx_);

if (ws.length() > 0)

{

//Inserted by dividing one line of CSV File to vector<String>.

std::vector<String> v1;

std::wstring column_data;

std::wistringstream line_data(ws);

while (std::getline(line_data, column_data, L','))

{

v1.push_back(column_data.c_str());

}

switch (line_num)

{

case 0:

create_column(v1);

break;

default:

create_line(v1);

}

void threads_join()

{

//Wait for the termination of all threads.

for (auto& th: vth_)

{

th.join();

}

};

int _tmain(int argc, _TCHAR* argv[])

{

std::chrono::system_clock::time_point now = std::chrono::system_clock::now();

_Tcsv_to_memtable memtable_;

UnicodeString argv_ = argv[1]; //Specify CSV Fire for input.

UnicodeString arg_o_ = argv[2]; //JSON output destination specification made with "TFDMemTable".

if (FileExists(argv_))

{

std::wfstream file_;

file_.open(argv_.w_str(), std::ios::in);

//The CSV file is utf8 format.

file_.imbue(std::locale(std::locale::empty(), new std::codecvt_utf8<wchar_t>));

std::wstring buff;

int line_num_{0};

while (std::getline(file_, buff))

{

//The append processing to "TFDMemTable" threads line by line.

memtable_.vth_.push_back(std::thread([&memtable_, &buff, line_num_](){

memtable_.create_line(buff, line_num_);

}));

//Since the command line is multibyte, AnsiString conversion is done.

AnsiString as_ = buff.c_str();

std::cout << as_ << std::endl;

++line_num_;

}

memtable_.threads_join();

//Finally, save the TFDMemTable contents as a JSON file.

memtable_.mem_table_->SaveToFile(arg_o_, TFDStorageFormat::sfJSON);

}

auto keika = std::chrono::system_clock::now() - now;

std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(keika).count();

std::cout << " millisecond." << std::endl ;

return 0;

}

There is a project file in github, you can try it right away.
GitHub – mojeld/csvfstream_to_TFDMemTable: Create json file of TFDMemTable format from CSV file.

f:id:mojeld:20170901174729p:plain

Reduce development time and get to market faster with RAD Studio, Delphi, or C++Builder.
Design. Code. Compile. Deploy.
Start Free Trial Upgrade Today

Free Delphi Community Edition Free C++Builder Community Edition

Cookie	Duration	Description
cookielawinfo-checkbox-analytics	11 months	This cookie is set by GDPR Cookie Consent plugin. The cookie is used to store the user consent for the cookies in the category "Analytics".
cookielawinfo-checkbox-functional	11 months	The cookie is set by GDPR cookie consent to record the user consent for the cookies in the category "Functional".
cookielawinfo-checkbox-necessary	11 months	This cookie is set by GDPR Cookie Consent plugin. The cookies is used to store the user consent for the cookies in the category "Necessary".
cookielawinfo-checkbox-others	11 months	This cookie is set by GDPR Cookie Consent plugin. The cookie is used to store the user consent for the cookies in the category "Other.
cookielawinfo-checkbox-performance	11 months	This cookie is set by GDPR Cookie Consent plugin. The cookie is used to store the user consent for the cookies in the category "Performance".
viewed_cookie_policy	11 months	The cookie is set by the GDPR Cookie Consent plugin and is used to store whether or not user has consented to the use of cookies. It does not store any personal data.

Convert CSV file to TFDMemTable format JSON file.

Convert from CSV file to “JSON file” of “TFDMemTable format”.

include

code

Leave a ReplyCancel reply

Search

Something Fresh

What Are The Container Adapters in Modern C++?

Delphi App For Go-Kart Racing Tops Both App Stores!

RAD Studio 12.1 Athens Patch 1 Available

Popular Posts

RAD Studio 12.1 Athens Patch 1 Available

Announcing the Availability of RAD Studio 12.1 Athens

Is Delphi A Memory Safe Language?

GetIt Update: RAD Studio 11 GetIt Online Installation along with Delphi and C++Builder Community Edition Are Now Available

GetIt Update: RAD Studio 12 GetIt Online Installation Is Now Available

Categories

Unknown Feed

Unknown Feed

Categories

Useful Links

Follow us

Convert CSV file to TFDMemTable format JSON file.

Convert from CSV file to “JSON file” of “TFDMemTable format”.

include

code

Leave a ReplyCancel reply

Join Our Global Developer Community

Search

Something Fresh

Popular Posts

Categories

Categories

Useful Links

Follow us