OR-Tools  8.1
file_util.h
Go to the documentation of this file.
1 // Copyright 2010-2018 Google LLC
2 // Licensed under the Apache License, Version 2.0 (the "License");
3 // you may not use this file except in compliance with the License.
4 // You may obtain a copy of the License at
5 //
6 // http://www.apache.org/licenses/LICENSE-2.0
7 //
8 // Unless required by applicable law or agreed to in writing, software
9 // distributed under the License is distributed on an "AS IS" BASIS,
10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 // See the License for the specific language governing permissions and
12 // limitations under the License.
13 
14 #ifndef OR_TOOLS_UTIL_FILE_UTIL_H_
15 #define OR_TOOLS_UTIL_FILE_UTIL_H_
16 
17 #include <limits>
18 #include <vector>
19 
20 #include "absl/status/statusor.h"
21 #include "absl/strings/string_view.h"
22 #include "google/protobuf/message.h"
23 #include "ortools/base/file.h"
24 #include "ortools/base/recordio.h"
25 
26 namespace operations_research {
27 
28 // Reads a file, optionally gzipped, to a string.
29 absl::StatusOr<std::string> ReadFileToString(absl::string_view filename);
30 
31 // Reads a proto from a file. Supports the following formats: binary, text,
32 // JSON, all of those optionally gzipped. Crashes on filesystem failures, e.g.
33 // file unreadable. Returns false on format failures, e.g. the file could be
34 // read, but the contents couldn't be parsed -- or maybe it was a valid JSON,
35 // text proto, or binary proto, but not of the right proto message.
36 // Returns true on success.
37 bool ReadFileToProto(absl::string_view filename,
38  google::protobuf::Message* proto);
39 
40 template <typename Proto>
41 Proto ReadFileToProtoOrDie(absl::string_view filename) {
42  Proto proto;
43  CHECK(ReadFileToProto(filename, &proto)) << "with file: '" << filename << "'";
44  return proto;
45 }
46 
47 enum class ProtoWriteFormat { kProtoText, kProtoBinary, kJson };
48 
49 // Writes a proto to a file. Supports the following formats: binary, text, JSON,
50 // all of those optionally gzipped. Returns false on failure.
51 // If 'proto_write_format' is kProtoBinary, ".bin" is appended to file_name. If
52 // 'proto_write_format' is kJson, ".json" is appended to file_name. If 'gzipped'
53 // is true, ".gz" is appended to file_name.
54 bool WriteProtoToFile(absl::string_view filename,
55  const google::protobuf::Message& proto,
56  ProtoWriteFormat proto_write_format, bool gzipped = false,
57  bool append_extension_to_file_name = true);
58 
59 namespace internal {
60 // General method to read expected_num_records from a file. If
61 // expected_num_records is -1, then reads all records from the file. If not,
62 // dies if the file doesn't contain exactly expected_num_records.
63 template <typename Proto>
64 std::vector<Proto> ReadNumRecords(File* file, int expected_num_records) {
66  std::vector<Proto> protos;
67  Proto proto;
68  int num_read = 0;
69  while (num_read != expected_num_records &&
70  reader.ReadProtocolMessage(&proto)) {
71  protos.push_back(proto);
72  ++num_read;
73  }
74 
75  CHECK(reader.Close())
76  << "File '" << file->filename()
77  << "'was not fully read, or something went wrong when closing "
78  "it. Is it the right format? (RecordIO of Protocol Buffers).";
79 
80  if (expected_num_records >= 0) {
81  CHECK_EQ(num_read, expected_num_records)
82  << "There were less than the expected " << expected_num_records
83  << " in the file.";
84  }
85 
86  return protos;
87 }
88 
89 // Ditto, taking a filename as argument.
90 template <typename Proto>
91 std::vector<Proto> ReadNumRecords(absl::string_view filename,
92  int expected_num_records) {
93  return ReadNumRecords<Proto>(file::OpenOrDie(filename, "r", file::Defaults()),
94  expected_num_records);
95 }
96 } // namespace internal
97 
98 // Reads all records in Proto format in 'file'. Silently does nothing if the
99 // file is empty. Dies if the file doesn't exist or contains something else than
100 // protos encoded in RecordIO format.
101 template <typename Proto>
102 std::vector<Proto> ReadAllRecordsOrDie(absl::string_view filename) {
103  return internal::ReadNumRecords<Proto>(filename, -1);
104 }
105 template <typename Proto>
106 std::vector<Proto> ReadAllRecordsOrDie(File* file) {
107  return internal::ReadNumRecords<Proto>(file, -1);
108 }
109 
110 // Reads one record from file, which must be in RecordIO binary proto format.
111 // Dies if the file can't be read, doesn't contain exactly one record, or
112 // contains something else than the expected proto in RecordIO format.
113 template <typename Proto>
114 Proto ReadOneRecordOrDie(absl::string_view filename) {
115  Proto p;
116  p.Swap(&internal::ReadNumRecords<Proto>(filename, 1)[0]);
117  return p;
118 }
119 
120 // Writes all records in Proto format to 'file'. Dies if it is unable to open
121 // the file or write to it.
122 template <typename Proto>
123 void WriteRecordsOrDie(absl::string_view filename,
124  const std::vector<Proto>& protos) {
125  recordio::RecordWriter writer(
126  file::OpenOrDie(filename, "w", file::Defaults()));
127  for (const Proto& proto : protos) {
129  }
130  CHECK(writer.Close());
131 }
132 
133 } // namespace operations_research
134 
135 #endif // OR_TOOLS_UTIL_FILE_UTIL_H_
operations_research::ReadFileToString
absl::StatusOr< std::string > ReadFileToString(absl::string_view filename)
Definition: file_util.cc:30
operations_research::ReadAllRecordsOrDie
std::vector< Proto > ReadAllRecordsOrDie(absl::string_view filename)
Definition: file_util.h:102
recordio.h
operations_research::ProtoWriteFormat::kProtoText
@ kProtoText
operations_research
The vehicle routing library lets one model and solve generic vehicle routing problems ranging from th...
Definition: dense_doubly_linked_list.h:21
recordio::RecordWriter
Definition: recordio.h:33
recordio::RecordWriter::WriteProtocolMessage
bool WriteProtocolMessage(const P &proto)
Definition: recordio.h:41
file.h
operations_research::internal::ReadNumRecords
std::vector< Proto > ReadNumRecords(File *file, int expected_num_records)
Definition: file_util.h:64
File
Definition: base/file.h:32
recordio::RecordReader::Close
bool Close()
Definition: recordio.cc:54
operations_research::WriteProtoToFile
bool WriteProtoToFile(absl::string_view filename, const google::protobuf::Message &proto, ProtoWriteFormat proto_write_format, bool gzipped, bool append_extension_to_file_name)
Definition: file_util.cc:77
CHECK_EQ
#define CHECK_EQ(val1, val2)
Definition: base/logging.h:697
recordio::RecordReader::ReadProtocolMessage
bool ReadProtocolMessage(P *const proto)
Definition: recordio.h:91
operations_research::ProtoWriteFormat
ProtoWriteFormat
Definition: file_util.h:47
operations_research::ReadFileToProto
bool ReadFileToProto(absl::string_view filename, google::protobuf::Message *proto)
Definition: file_util.cc:37
file::Defaults
int Defaults()
Definition: base/file.h:119
file
Definition: file.cc:141
operations_research::ReadOneRecordOrDie
Proto ReadOneRecordOrDie(absl::string_view filename)
Definition: file_util.h:114
recordio::RecordWriter::Close
bool Close()
Definition: recordio.cc:29
internal
Definition: bop_parameters.pb.h:39
proto
CpModelProto proto
Definition: cp_model_fz_solver.cc:107
operations_research::WriteRecordsOrDie
void WriteRecordsOrDie(absl::string_view filename, const std::vector< Proto > &protos)
Definition: file_util.h:123
file::OpenOrDie
File * OpenOrDie(const absl::string_view &filename, const absl::string_view &mode, int flags)
Definition: file.cc:154
operations_research::ReadFileToProtoOrDie
Proto ReadFileToProtoOrDie(absl::string_view filename)
Definition: file_util.h:41
recordio::RecordReader
Definition: recordio.h:86
CHECK
#define CHECK(condition)
Definition: base/logging.h:495