Line data Source code
1 : // DUNE DAQ modification notice:
2 : // This file has been modified from the original ATLAS oks source for the DUNE DAQ project.
3 : // Fork baseline commit: oks-08-03-04 (2022-04-14).
4 : // Renamed since fork: no.
5 :
6 : #define _OksBuildDll_
7 :
8 : #include <errno.h>
9 : #include <stdio.h>
10 : #include <stdlib.h>
11 : #include <string.h>
12 : #include <unistd.h>
13 : #include <wchar.h>
14 : // #include <security/pam_appl.h>
15 : #include <sys/stat.h>
16 : #include <sys/wait.h>
17 :
18 : #include <algorithm>
19 : #include <chrono>
20 : #include <cstring>
21 : #include <ctime>
22 : #include <filesystem>
23 : #include <fstream>
24 : #include <mutex>
25 : #include <sstream>
26 : #include <stdexcept>
27 : #include <vector>
28 :
29 : #include "ers/ers.hpp"
30 : #include "logging/Logging.hpp"
31 :
32 : #include "okssystem/Host.hpp"
33 : #include "okssystem/User.hpp"
34 : #include "okssystem/exceptions.hpp"
35 :
36 : // #include <daq_tokens/verify.h>
37 :
38 : #include "oks/attribute.hpp"
39 : #include "oks/class.hpp"
40 : #include "oks/cstring.hpp"
41 : #include "oks/file.hpp"
42 : #include "oks/kernel.hpp"
43 : #include "oks/method.hpp"
44 : #include "oks/object.hpp"
45 : #include "oks/pipeline.hpp"
46 : #include "oks/profiler.hpp"
47 : #include "oks/relationship.hpp"
48 : #include "oks/xml.hpp"
49 :
50 : #include "oks_utils.hpp"
51 :
52 : namespace dunedaq {
53 : namespace oks {
54 :
55 : std::mutex OksKernel::p_parallel_out_mutex;
56 :
57 : static std::mutex s_get_cwd_mutex;
58 :
59 : bool OksKernel::p_skip_string_range = false;
60 : bool OksKernel::p_use_strict_repository_paths = true;
61 : char* OksKernel::s_cwd = nullptr;
62 :
63 : std::string OksKernel::p_repository_root;
64 : std::string OksKernel::p_repository_mapping_dir;
65 :
66 : int OksKernel::p_threads_pool_size = 0;
67 :
68 : class GitFoldersHolder
69 : {
70 : public:
71 41 : ~GitFoldersHolder()
72 : {
73 41 : std::lock_guard scoped_lock(s_git_folders_mutex);
74 :
75 41 : for (const auto& x : s_git_folders)
76 0 : remove(x);
77 :
78 41 : s_git_folders.clear();
79 41 : }
80 :
81 0 : static void remove(const std::string& path)
82 : {
83 0 : try {
84 0 : std::filesystem::remove_all(path);
85 0 : } catch (std::exception& ex) {
86 0 : Oks::error_msg("OksKernel::~OksKernel")
87 0 : << "cannot remove user repository \"" << path << "\"" << ex.what() << std::endl;
88 0 : }
89 0 : }
90 :
91 0 : void insert(const std::string& path)
92 : {
93 0 : std::lock_guard scoped_lock(s_git_folders_mutex);
94 0 : s_git_folders.insert(path);
95 0 : }
96 :
97 0 : void erase(const std::string& path)
98 : {
99 0 : std::lock_guard scoped_lock(s_git_folders_mutex);
100 0 : s_git_folders.erase(path);
101 0 : }
102 :
103 : private:
104 : std::mutex s_git_folders_mutex;
105 : std::set<std::string> s_git_folders;
106 : };
107 :
108 : static GitFoldersHolder s_git_folders;
109 :
110 : const std::string
111 285 : strerror(int error)
112 : {
113 285 : char buffer[1024];
114 285 : buffer[0] = 0;
115 285 : return std::string(strerror_r(error, buffer, 1024));
116 : }
117 :
118 41 : ERS_DECLARE_ISSUE(kernel,
119 : SetGroupIdFailed,
120 : "cannot set group ID " << id << " for the file \'" << file << "\': chown() failed with code " << code
121 : << " , reason = \'" << why << '\'',
122 : ((long)id)((const char*)file)((int)code)((std::string)why))
123 :
124 41 : ERS_DECLARE_ISSUE(kernel, BindError, "Found unresolved reference(s):\n" << which, ((std::string)which))
125 :
126 41 : ERS_DECLARE_ISSUE(kernel,
127 : ClassAlreadyDefined,
128 : "class \"" << name << "\" is defined in files \"" << f1 << "\" and \"" << f2 << "\"",
129 : ((std::string)name)((std::string)f1)((std::string)f2))
130 :
131 41 : ERS_DECLARE_ISSUE(kernel, InternalError, "internal error: " << text, ((std::string)text))
132 :
133 : std::string
134 0 : CanNotOpenFile::fill(const char* prefix, const std::string& name, const std::string& reason) noexcept
135 : {
136 0 : std::string result(prefix);
137 0 : result += "(): ";
138 :
139 0 : if (name.empty())
140 0 : return (result + "file name is empty");
141 :
142 0 : result += "cannot load file \'" + name + '\'';
143 :
144 0 : if (!reason.empty()) {
145 0 : result += " because:\n" + reason;
146 : }
147 :
148 0 : return result;
149 0 : }
150 :
151 : std::string
152 0 : FailedLoadFile::fill(const std::string& what, const std::string& name, const std::string& reason) noexcept
153 : {
154 0 : return (std::string("Failed to load ") + what + " \"" + name + "\" because:\n" + reason);
155 : }
156 :
157 : std::string
158 0 : FailedReloadFile::fill(const std::string& names, const std::string& reason) noexcept
159 : {
160 0 : return (std::string("Failed to re-load file(s) ") + names + " because:\n" + reason);
161 : }
162 :
163 : std::string
164 0 : RepositoryOperationFailed::fill(const char* op, const std::string& reason) noexcept
165 : {
166 0 : return (std::string("Failed to ") + op + " file(s) because:\n" + reason);
167 : }
168 :
169 : std::string
170 0 : CanNotCreateFile::fill(const char* prefix,
171 : const char* what,
172 : const std::string& name,
173 : const std::string& reason) noexcept
174 : {
175 0 : return std::string(prefix) + "(): failed to create new " + what + " \'" + name + "\' because:\n" + reason;
176 : }
177 :
178 : std::string
179 0 : CanNotCreateRepositoryDir::fill(const char* prefix, const std::string& name) noexcept
180 : {
181 0 : return std::string(prefix) + "(): failed to create user repository dir, mkdtemp(\'" + name +
182 0 : "\') failed: " + strerror(errno);
183 : }
184 :
185 : std::string
186 0 : CanNotWriteToFile::fill(const char* prefix,
187 : const char* item,
188 : const std::string& name,
189 : const std::string& reason) noexcept
190 : {
191 0 : return (std::string(prefix) + "(): failed to write " + item + " to \'" + name + "\' because:\n" + reason);
192 : }
193 :
194 : std::string
195 0 : CanNotBackupFile::fill(const std::string& name, const std::string& reason) noexcept
196 : {
197 0 : return (std::string("Failed to make a backup file of \'") + name + "\' because:\n" + reason);
198 : }
199 :
200 : std::string
201 0 : CanNotSetActiveFile::fill(const char* item, const std::string& name, const std::string& reason) noexcept
202 : {
203 0 : return (std::string("Failed to set active ") + item + " file \'" + name + "\' because:\n" + reason);
204 : }
205 :
206 : std::string
207 0 : CanNotSetFile::fill(const OksClass* c, const OksObject* o, const OksFile& file, const std::string& reason) noexcept
208 : {
209 0 : std::ostringstream s;
210 0 : s << "Failed to move ";
211 0 : if (c)
212 0 : s << "class " << c->get_name();
213 : else
214 0 : s << "object " << o;
215 0 : s << " to file \'" << file.get_full_file_name() << "\' because:\n" << reason;
216 0 : return s.str();
217 0 : }
218 :
219 : std::string
220 0 : CannotAddClass::fill(const OksClass& c, const std::string& reason) noexcept
221 : {
222 0 : return (std::string("Cannot add class \'") + c.get_name() + "\' because:\n" + reason);
223 : }
224 :
225 : std::string
226 285 : CannotResolvePath::fill(const std::string& path, int error_code) noexcept
227 : {
228 285 : std::ostringstream text;
229 285 : text << "realpath(\'" << path << "\') has failed with code " << error_code << ": \'" << strerror(error_code) << '\'';
230 285 : return text.str();
231 285 : }
232 :
233 : static const std::string&
234 0 : get_temporary_dir()
235 : {
236 0 : static std::string s_dir;
237 0 : static std::once_flag flag;
238 :
239 0 : std::call_once(flag, []() {
240 0 : try {
241 0 : if (!getenv("OKS_GIT_NO_VAR_DIR")) {
242 0 : OksSystem::User myself;
243 :
244 0 : s_dir = "/var/run/user/";
245 0 : s_dir.append(std::to_string(myself.identity()));
246 :
247 0 : if (std::filesystem::is_directory(s_dir) == false) {
248 0 : TLOG_DEBUG(1) << "directory " << s_dir << " does not exist";
249 0 : s_dir.clear();
250 : } else {
251 0 : const std::filesystem::space_info si = std::filesystem::space(s_dir);
252 0 : const double available = static_cast<double>(si.available) / static_cast<double>(si.capacity);
253 :
254 0 : if (available < 0.5) {
255 0 : TLOG_DEBUG(1) << "usage of " << s_dir << " is " << static_cast<int>((1.0 - available) * 100.0)
256 0 : << "%, will use standard temporary path";
257 0 : s_dir.clear();
258 : }
259 : }
260 0 : }
261 :
262 0 : if (s_dir.empty())
263 0 : s_dir = std::filesystem::temp_directory_path().string();
264 0 : } catch (std::exception& ex) {
265 0 : Oks::error_msg("OksKernel::OksKernel") << "cannot get temporary directory path: " << ex.what() << std::endl;
266 0 : s_dir = "/tmp";
267 0 : }
268 0 : });
269 :
270 0 : return s_dir;
271 : }
272 :
273 : const char*
274 0 : OksKernel::GetVersion()
275 : {
276 0 : return "862f2957270";
277 : }
278 :
279 : const std::string&
280 410 : OksKernel::get_repository_root()
281 : {
282 410 : static std::once_flag flag;
283 :
284 410 : std::call_once(flag, []() {
285 19 : if (const char* s = getenv("TDAQ_DB_REPOSITORY")) {
286 0 : std::string rep(s);
287 :
288 0 : if (!std::all_of(rep.begin(), rep.end(), [](char c) { return std::isspace(c); })) {
289 0 : const char* p = getenv("OKS_GIT_PROTOCOL");
290 :
291 0 : if (p && !*p)
292 0 : p = nullptr;
293 :
294 0 : Oks::Tokenizer t(rep, "|");
295 0 : std::string token;
296 :
297 0 : while (t.next(token) && p_repository_root.empty()) {
298 0 : if (p) {
299 0 : if (token.find(p) == 0) {
300 0 : p_repository_root = token;
301 : }
302 : } else {
303 0 : p_repository_root = token;
304 : }
305 : }
306 :
307 0 : if (p_repository_root.empty())
308 0 : Oks::error_msg("OksKernel::OksKernel")
309 0 : << "cannot find OKS_GIT_PROTOCOL=\"" << p << "\" in TDAQ_DB_REPOSITORY=\"" << rep << '\"' << std::endl;
310 0 : }
311 0 : }
312 19 : });
313 :
314 410 : return p_repository_root;
315 : }
316 :
317 : const std::string&
318 0 : OksKernel::get_repository_mapping_dir()
319 : {
320 0 : static std::once_flag flag;
321 :
322 0 : std::call_once(flag, []() {
323 0 : if (const char* s = getenv("OKS_REPOSITORY_MAPPING_DIR")) {
324 0 : p_repository_mapping_dir = s;
325 0 : Oks::real_path(p_repository_mapping_dir, true);
326 : }
327 0 : });
328 :
329 0 : return p_repository_mapping_dir;
330 : }
331 :
332 : ////////////////////////////////////////////////////////////////////////////////
333 :
334 : const std::string&
335 1632 : OksKernel::get_user_repository_root() const
336 : {
337 1632 : if (p_allow_repository) {
338 0 : OksKernel* k(const_cast<OksKernel*>(this));
339 0 : if (!p_user_repository_root_inited) {
340 0 : if (const char* s = getenv("TDAQ_DB_USER_REPOSITORY")) {
341 0 : if (s[0] != 0) {
342 0 : k->set_user_repository_root(s);
343 : }
344 : }
345 0 : k->p_user_repository_root_inited = true;
346 : }
347 : }
348 :
349 1632 : return p_user_repository_root;
350 : }
351 :
352 : void
353 0 : OksKernel::set_user_repository_root(const std::string& path, const std::string& version)
354 : {
355 0 : if (get_repository_root().empty()) {
356 0 : TLOG_DEBUG(1) << "Failed to set user-repository-root:\n\tcaused by: repository-root is not set";
357 0 : p_user_repository_root.clear();
358 0 : return;
359 : }
360 :
361 0 : if (path.empty()) {
362 0 : p_user_repository_root.clear();
363 0 : return;
364 : }
365 :
366 0 : std::string s;
367 0 : s = path;
368 :
369 0 : try {
370 0 : Oks::real_path(s, false);
371 0 : } catch (exception& ex) {
372 0 : TLOG_DEBUG(1) << "Failed to set user-repository-root = \'" << s << "\':\n\tcaused by: " << ex.what();
373 0 : }
374 :
375 0 : for (std::string::size_type idx = s.size() - 1; idx > 0 && s[idx] == '/'; idx--) {
376 0 : s.erase(idx);
377 : }
378 :
379 0 : p_allow_repository = true;
380 :
381 0 : p_user_repository_root = s;
382 0 : p_user_repository_root_inited = true;
383 :
384 : // is used for backup restore by RDB
385 0 : if (!version.empty())
386 0 : p_repository_version = version;
387 0 : }
388 :
389 : ////////////////////////////////////////////////////////////////////////////////
390 :
391 : std::string&
392 45 : OksKernel::get_host_name()
393 : {
394 64 : static std::string s_host_name;
395 45 : static std::once_flag flag;
396 :
397 45 : std::call_once(flag, []() {
398 19 : s_host_name = OksSystem::LocalHost::instance()->full_local_name();
399 :
400 19 : if (s_host_name.empty())
401 0 : s_host_name = "unknown.host";
402 19 : });
403 :
404 45 : return s_host_name;
405 : }
406 :
407 : std::string&
408 0 : OksKernel::get_domain_name()
409 : {
410 0 : static std::string s_domain_name;
411 0 : static std::once_flag flag;
412 :
413 0 : std::call_once(flag, []() {
414 0 : std::string::size_type idx = get_host_name().find('.');
415 :
416 0 : if (idx != std::string::npos)
417 0 : s_domain_name = get_host_name().substr(idx + 1);
418 :
419 0 : if (s_domain_name.empty())
420 0 : s_domain_name = "unknown.domain";
421 0 : });
422 :
423 0 : return s_domain_name;
424 : }
425 :
426 : std::string&
427 45 : OksKernel::get_user_name()
428 : {
429 64 : static std::string s_user_name;
430 45 : static std::once_flag flag;
431 :
432 45 : std::call_once(flag, []() {
433 19 : OksSystem::User myself;
434 19 : s_user_name = myself.name_safe();
435 :
436 19 : if (s_user_name.empty())
437 0 : s_user_name = "unknown.user";
438 19 : });
439 :
440 45 : return s_user_name;
441 : }
442 :
443 : ////////////////////////////////////////////////////////////////////////////////
444 :
445 : static long
446 315 : get_file_length(std::ifstream& f)
447 : {
448 315 : long file_length = 0;
449 :
450 : // get size of file and check that it is not empty
451 :
452 315 : f.seekg(0, std::ios::end);
453 :
454 : #if __GNUC__ >= 3
455 315 : file_length = static_cast<std::streamoff>(f.tellg());
456 : #else
457 : file_length = f.tellg();
458 : #endif
459 :
460 315 : f.seekg(0, std::ios::beg);
461 :
462 315 : return file_length;
463 : }
464 :
465 : // makes name of function with parameters for methods working with files
466 :
467 : std::string
468 1104 : make_fname(const char* f, size_t f_len, const std::string& file, bool* p, const OksFile* const* fh)
469 : {
470 1104 : const char _prefix[] = "OksKernel::";
471 1104 : std::string fname(_prefix, (sizeof(_prefix) - 1));
472 :
473 1104 : fname.append(f, f_len);
474 :
475 1104 : const char _x1[] = "(file \'";
476 1104 : fname.append(_x1, (sizeof(_x1) - 1));
477 1104 : fname.append(file);
478 1104 : fname.push_back('\'');
479 :
480 : // add boolean parameter
481 :
482 1104 : if (p) {
483 595 : if (*p) {
484 100 : const char _true_str[] = ", true";
485 100 : fname.append(_true_str, (sizeof(_true_str) - 1));
486 : } else {
487 495 : const char _false_str[] = ", false";
488 495 : fname.append(_false_str, (sizeof(_false_str) - 1));
489 : }
490 : }
491 :
492 : // add file handler parameter
493 :
494 1104 : if (fh && *fh) {
495 744 : const char _x2[] = ", included by file \'";
496 744 : fname.append(_x2, (sizeof(_x2) - 1));
497 744 : fname.append((*fh)->get_full_file_name());
498 744 : fname.push_back('\'');
499 : }
500 :
501 1104 : fname.push_back(')');
502 :
503 1104 : return fname;
504 0 : }
505 :
506 : //
507 : // Prototypes for error and warning messages
508 : //
509 :
510 : std::ostream&
511 0 : Oks::error_msg(const char* msg)
512 : {
513 0 : std::cerr << "ERROR [" << msg << "]:\n";
514 0 : return std::cerr;
515 : }
516 :
517 : std::ostream&
518 0 : Oks::warning_msg(const char* msg)
519 : {
520 0 : std::cerr << "WARNING [" << msg << "]:\n";
521 0 : return std::cerr;
522 : }
523 :
524 : void
525 21709 : Oks::substitute_variables(std::string& s)
526 : {
527 21709 : std::string::size_type pos = 0; // position of tested string index
528 21709 : std::string::size_type p_start = 0; // begining of variable
529 21709 : std::string::size_type p_end = 0; // begining of variable
530 :
531 21709 : while (((p_start = s.find("$(", pos)) != std::string::npos) &&
532 0 : ((p_end = s.find(")", p_start + 2)) != std::string::npos)) {
533 0 : std::string var(s, p_start + 2, p_end - p_start - 2);
534 :
535 0 : char* env = getenv(var.c_str());
536 :
537 0 : if (env) {
538 0 : s.replace(p_start, p_end - p_start + 1, env);
539 : }
540 :
541 0 : pos = p_start + 1;
542 0 : }
543 21709 : }
544 :
545 : bool
546 21337 : Oks::real_path(std::string& path, bool ignore_errors)
547 : {
548 21337 : char resolved_name[PATH_MAX];
549 :
550 21337 : if (realpath(path.c_str(), resolved_name) != 0) {
551 7249 : path = resolved_name;
552 7249 : return true;
553 : } else {
554 14088 : if (ignore_errors) {
555 13803 : TLOG_DEBUG(3) << "realpath(\'" << path << "\') has failed with code " << errno << ": \'" << strerror(errno)
556 13803 : << '\'';
557 13803 : return false;
558 : } else {
559 285 : throw CannotResolvePath(path, errno);
560 : }
561 : }
562 : }
563 :
564 : unsigned long OksKernel::p_count = 0;
565 : const char OksNameTable::symbols[] = "0123456789"
566 : "abcdefghijklmnoprstuvwxyz"
567 : "ABCDEFGHIJKLMNOPRSTUVWXYZ";
568 :
569 : OksString*
570 0 : OksNameTable::get()
571 : {
572 0 : static const size_t slen(sizeof(symbols) - 1);
573 0 : static const size_t slen2(slen * slen);
574 :
575 0 : char b[16], *buf = b;
576 :
577 0 : count++;
578 :
579 0 : *buf++ = symbols[count % slen];
580 :
581 0 : if ((size_t)count >= slen) {
582 0 : *buf++ = symbols[(count / slen) % slen];
583 :
584 0 : if ((size_t)count >= slen2) {
585 0 : *buf++ = symbols[(count / slen2) % slen];
586 : }
587 : }
588 :
589 0 : *buf = '\0';
590 :
591 0 : return new OksString(b);
592 : }
593 :
594 133 : OksAliasTable::~OksAliasTable()
595 : {
596 133 : if (p_aliases.empty())
597 133 : return;
598 :
599 0 : for (auto& i : p_aliases) {
600 0 : if (OksString* s = i.second->class_name)
601 0 : delete s;
602 :
603 0 : delete i.second;
604 0 : delete const_cast<OksString*>(i.first);
605 : }
606 :
607 0 : p_aliases.clear();
608 133 : }
609 :
610 : std::string
611 0 : OksKernel::get_tmp_file(const std::string& file)
612 : {
613 0 : unsigned int j = 0;
614 0 : std::string s2;
615 0 : std::ostringstream s;
616 :
617 0 : while (j++ < 1000000) {
618 0 : s.str("");
619 0 : s << file << '.' << get_user_name() << ':' << get_host_name() << ':' << getpid() << ':' << j;
620 :
621 0 : s2 = s.str();
622 :
623 0 : std::ofstream f(s2.c_str(), std::ios::in);
624 :
625 0 : if (!f)
626 0 : return s2;
627 0 : }
628 :
629 0 : std::string s3(file);
630 0 : s3 += ".tmp";
631 :
632 0 : return s3;
633 0 : }
634 :
635 : const char*
636 95 : OksKernel::get_cwd()
637 : {
638 95 : std::lock_guard scoped_lock(s_get_cwd_mutex);
639 :
640 95 : if (!s_cwd) {
641 95 : errno = 0;
642 95 : long size = pathconf(".", _PC_PATH_MAX);
643 95 : if (errno) {
644 0 : Oks::error_msg("OksKernel::OksKernel") << "pathconf(\".\", _PC_PATH_MAX) has failed with code " << errno << ": \'"
645 0 : << strerror(errno) << '\'' << std::endl;
646 : } else {
647 95 : if (size == -1) {
648 0 : size = PATH_MAX;
649 : }
650 95 : s_cwd = new char[size];
651 95 : if (!getcwd(s_cwd, (size_t)size)) {
652 0 : Oks::error_msg("OksKernel::OksKernel")
653 0 : << "getcwd() has failed with code " << errno << ": \'" << strerror(errno) << '\'' << std::endl;
654 0 : delete[] s_cwd;
655 0 : s_cwd = 0;
656 : }
657 : }
658 :
659 95 : if (!s_cwd) {
660 0 : s_cwd = new char[2];
661 0 : s_cwd[0] = '/';
662 0 : s_cwd[1] = 0;
663 : }
664 :
665 95 : TLOG_DEBUG(1) << "Current working dir: \'" << s_cwd << '\'';
666 : }
667 :
668 95 : return s_cwd;
669 95 : }
670 :
671 : /******************************************************************************/
672 : /***************************** OKS Kernel Class *****************************/
673 : /******************************************************************************/
674 :
675 95 : OksKernel::OksKernel(bool sm, bool vm, bool tm, bool allow_repository, const char* version, std::string branch_name)
676 95 : : p_silence(sm)
677 95 : , p_verbose(vm)
678 95 : , p_profiling(tm)
679 95 : , p_allow_repository(allow_repository)
680 95 : , p_allow_duplicated_classes(true)
681 95 : , p_allow_duplicated_objects(false)
682 95 : , p_test_duplicated_objects_via_inheritance(false)
683 95 : , p_user_repository_root_inited(false)
684 95 : , p_user_repository_root_created(false)
685 95 : , p_active_schema(nullptr)
686 95 : , p_active_data(nullptr)
687 95 : , p_close_all(false)
688 95 : , profiler(nullptr)
689 95 : , p_create_object_notify_fn(nullptr)
690 95 : , p_create_object_notify_param(nullptr)
691 95 : , p_change_object_notify_fn(nullptr)
692 95 : , p_change_object_notify_param(nullptr)
693 95 : , p_delete_object_notify_fn(nullptr)
694 95 : , p_delete_object_notify_param(nullptr)
695 : {
696 95 : OSK_VERBOSE_REPORT("Enter OksKernel::OksKernel(" << sm << ", " << vm << ", " << tm << ')')
697 :
698 95 : struct __InitFromEnv__
699 : {
700 : const char* name;
701 : bool& value;
702 95 : } vars[] = { { "OKS_KERNEL_VERBOSE", p_verbose },
703 95 : { "OKS_KERNEL_SILENCE", p_silence },
704 95 : { "OKS_KERNEL_PROFILING", p_profiling },
705 95 : { "OKS_KERNEL_ALLOW_REPOSITORY", p_allow_repository },
706 95 : { "OKS_KERNEL_ALLOW_DUPLICATED_CLASSES", p_allow_duplicated_classes },
707 95 : { "OKS_KERNEL_ALLOW_DUPLICATED_OBJECTS", p_allow_duplicated_objects },
708 95 : { "OKS_KERNEL_TEST_DUPLICATED_OBJECTS_VIA_INHERITANCE", p_test_duplicated_objects_via_inheritance },
709 95 : { "OKS_KERNEL_SKIP_STRING_RANGE", p_skip_string_range } };
710 :
711 855 : for (unsigned int i = 0; i < sizeof(vars) / sizeof(__InitFromEnv__); ++i) {
712 760 : if (char* s = getenv(vars[i].name)) {
713 0 : vars[i].value = (strcmp(s, "no")) ? true : false;
714 : }
715 : }
716 :
717 95 : {
718 95 : const char* oks_db_root = getenv("OKS_DB_ROOT");
719 :
720 95 : if (oks_db_root == nullptr || *oks_db_root == 0)
721 95 : oks_db_root = getenv("DUNEDAQ_DB_PATH");
722 : else
723 0 : p_use_strict_repository_paths = false;
724 :
725 95 : OSK_VERBOSE_REPORT("database root = \'" << oks_db_root << "\' (as defined by the OKS_DB_ROOT or DUNEDAQ_DB_PATH)");
726 :
727 : // if not defined, set oks_db_root to empty string (to avoid std::string constructor crash when string is 0)
728 :
729 95 : if (oks_db_root == nullptr)
730 0 : oks_db_root = "";
731 :
732 : // temporal set
733 :
734 95 : Oks::Tokenizer t(oks_db_root, ":");
735 95 : std::string token;
736 :
737 7220 : while (t.next(token))
738 7030 : insert_repository_dir(token);
739 95 : }
740 :
741 95 : {
742 95 : std::unique_lock lock(p_kernel_mutex);
743 :
744 95 : if (!p_count++) {
745 95 : OksClass::create_notify_fn = 0;
746 95 : OksClass::change_notify_fn = 0;
747 95 : OksClass::delete_notify_fn = 0;
748 : }
749 :
750 95 : get_cwd();
751 95 : }
752 :
753 95 : {
754 95 : static std::once_flag flag;
755 :
756 95 : std::call_once(flag, []() {
757 19 : if (char* s = getenv("OKS_KERNEL_THREADS_POOL_SIZE")) {
758 0 : if (*s != '\0') {
759 0 : p_threads_pool_size = atoi(s);
760 0 : if (p_threads_pool_size < 1)
761 0 : p_threads_pool_size = 1;
762 : }
763 : }
764 :
765 19 : if (!p_threads_pool_size) {
766 19 : errno = 0;
767 19 : p_threads_pool_size = sysconf(_SC_NPROCESSORS_ONLN);
768 19 : if (p_threads_pool_size == -1 && !errno) {
769 0 : Oks::error_msg("OksKernel::OksKernel()") << " sysconf(_SC_NPROCESSORS_ONLN) has failed with code " << errno
770 0 : << ": \'" << strerror(errno) << '\'' << std::endl;
771 0 : p_threads_pool_size = 0;
772 : }
773 : }
774 :
775 19 : if (!p_threads_pool_size)
776 0 : p_threads_pool_size = 2;
777 :
778 19 : TLOG_DEBUG(2) << "Threads pool size: " << p_threads_pool_size;
779 19 : });
780 : }
781 :
782 : // process OKS server URL settings if any
783 95 : if (p_allow_repository && !get_repository_root().empty()) {
784 0 : if (get_user_repository_root().empty()) {
785 : // parse specific version option if any
786 0 : std::string param, val;
787 :
788 0 : if (!version)
789 0 : version = getenv("TDAQ_DB_VERSION");
790 :
791 0 : if (version) {
792 0 : if (*version) {
793 0 : if (const char* value = strchr(version, ':')) {
794 0 : param.assign(version, value - version);
795 0 : val.assign(value + 1);
796 : } else
797 0 : Oks::error_msg("OksKernel::OksKernel")
798 : << "bad version value \"" << version
799 0 : << "\" expecting parameter:value format (check TDAQ_DB_VERSION variable)" << std::endl;
800 : }
801 : }
802 :
803 0 : try {
804 0 : std::string tmp_dirname = OksKernel::create_user_repository_dir();
805 :
806 0 : set_user_repository_root(tmp_dirname);
807 0 : p_user_repository_root_created = true;
808 0 : s_git_folders.insert(p_user_repository_root);
809 :
810 0 : if (branch_name.empty())
811 0 : if (const char* var = getenv("TDAQ_DB_BRANCH"))
812 0 : branch_name = var;
813 :
814 0 : k_checkout_repository(param, val, branch_name);
815 0 : } catch (exception& ex) {
816 0 : Oks::error_msg("OksKernel::OksKernel") << "cannot check out user repository: " << ex.what() << std::endl;
817 0 : }
818 0 : } else {
819 0 : if (!p_silence) {
820 0 : try {
821 0 : std::lock_guard lock(p_parallel_out_mutex);
822 0 : std::cout << "attach to repository \"" << get_user_repository_root() << "\" version "
823 0 : << read_repository_version() << std::endl;
824 0 : } catch (exception& ex) {
825 0 : Oks::error_msg("OksKernel::OksKernel") << "cannot read user repository version: " << ex.what() << std::endl;
826 0 : }
827 : }
828 : }
829 : }
830 :
831 : #ifndef ERS_NO_DEBUG
832 95 : if (p_profiling)
833 0 : profiler = (OksProfiler*)new OksProfiler();
834 : #endif
835 95 : }
836 :
837 0 : OksKernel::OksKernel(const OksKernel& src, bool copy_repository)
838 0 : : p_silence(src.p_silence)
839 0 : , p_verbose(src.p_verbose)
840 0 : , p_profiling(src.p_profiling)
841 0 : , p_allow_repository(src.p_allow_repository)
842 0 : , p_allow_duplicated_classes(src.p_allow_duplicated_classes)
843 0 : , p_allow_duplicated_objects(src.p_allow_duplicated_objects)
844 0 : , p_test_duplicated_objects_via_inheritance(src.p_test_duplicated_objects_via_inheritance)
845 0 : , p_user_repository_root(src.p_user_repository_root)
846 0 : , p_user_repository_root_inited(src.p_user_repository_root_inited)
847 0 : , p_user_repository_root_created(false)
848 0 : , p_repository_version(src.p_repository_version)
849 0 : , p_repository_checkout_ts(src.p_repository_checkout_ts)
850 0 : , p_repository_update_ts(src.p_repository_update_ts)
851 0 : , p_active_schema(nullptr)
852 0 : , p_active_data(nullptr)
853 0 : , p_close_all(false)
854 0 : , profiler(nullptr)
855 0 : , p_create_object_notify_fn(nullptr)
856 0 : , p_create_object_notify_param(nullptr)
857 0 : , p_change_object_notify_fn(nullptr)
858 0 : , p_change_object_notify_param(nullptr)
859 0 : , p_delete_object_notify_fn(nullptr)
860 0 : , p_delete_object_notify_param(nullptr)
861 : {
862 :
863 0 : {
864 0 : std::unique_lock lock(p_kernel_mutex);
865 0 : p_count++;
866 0 : }
867 :
868 0 : if (copy_repository && p_user_repository_root_inited) {
869 0 : try {
870 0 : p_user_repository_root = OksKernel::create_user_repository_dir();
871 0 : p_user_repository_root_created = true;
872 0 : Oks::real_path(p_user_repository_root, false);
873 0 : s_git_folders.insert(p_user_repository_root);
874 0 : k_copy_repository(src.p_user_repository_root, p_user_repository_root);
875 0 : } catch (exception& ex) {
876 0 : Oks::error_msg("OksKernel::OksKernel") << "cannot copy user repository: " << ex.what() << std::endl;
877 0 : remove_user_repository_dir();
878 0 : throw;
879 0 : }
880 :
881 0 : p_repository_dirs.push_back(p_user_repository_root);
882 : }
883 :
884 : // copy repository directories
885 0 : for (const auto& i : src.p_repository_dirs)
886 0 : p_repository_dirs.push_back(i);
887 :
888 : // copy schema and data files
889 0 : {
890 0 : const OksFile::Map* src_files[2] = { &src.p_schema_files, &src.p_data_files };
891 0 : OksFile::Map* dst_files[2] = { &p_schema_files, &p_data_files };
892 :
893 0 : for (int i = 0; i < 2; ++i) {
894 0 : for (const auto& j : *src_files[i]) {
895 0 : OksFile* f = new OksFile(*j.second);
896 0 : f->p_kernel = this;
897 0 : (*dst_files[i])[&f->p_full_name] = f;
898 : }
899 : }
900 : }
901 :
902 : // search a class in map is not efficient, if there are many such operations; use array with class index for fast
903 : // search
904 0 : OksClass** c_table = new OksClass*[src.p_classes.size()];
905 0 : unsigned int idx(0);
906 :
907 : // copy classes: first iteration to define class names and simple properties
908 0 : for (const auto& i : src.p_classes) {
909 0 : const OksClass& src_c(*i.second);
910 0 : OksClass* c = new OksClass(src_c.p_name,
911 0 : src_c.p_description,
912 0 : src_c.p_abstract,
913 : nullptr,
914 0 : src_c.p_transient); // pass 0 kernel to avoid class insertion
915 :
916 0 : c->p_kernel = this;
917 0 : p_classes[c->get_name().c_str()] = c;
918 :
919 0 : const_cast<OksClass&>(src_c).p_id = idx++;
920 0 : c->p_id = src_c.p_id;
921 0 : c_table[c->p_id] = c;
922 :
923 0 : c->p_abstract = src_c.p_abstract;
924 0 : c->p_to_be_deleted = src_c.p_to_be_deleted;
925 0 : c->p_instance_size = src_c.p_instance_size;
926 0 : c->p_file = (*p_schema_files.find(&src_c.p_file->p_full_name)).second;
927 0 : c->p_objects = (src_c.p_objects ? new OksObject::Map() : nullptr);
928 :
929 : // copy super-classes
930 0 : if (const std::list<std::string*>* scls = src_c.p_super_classes) {
931 0 : c->p_super_classes = new std::list<std::string*>();
932 :
933 0 : for (const auto& j : *scls)
934 0 : c->p_super_classes->push_back(new std::string(*j));
935 : }
936 :
937 : // copy direct attributes
938 0 : if (const std::list<OksAttribute*>* attrs = src_c.p_attributes) {
939 0 : c->p_attributes = new std::list<OksAttribute*>();
940 :
941 0 : for (const auto& j : *attrs) {
942 0 : OksAttribute* a = new OksAttribute(j->p_name, c);
943 :
944 0 : a->p_range = j->p_range;
945 0 : a->p_data_type = j->p_data_type;
946 0 : a->p_multi_values = j->p_multi_values;
947 0 : a->p_no_null = j->p_no_null;
948 0 : a->p_init_value = j->p_init_value;
949 0 : a->p_init_data = j->p_init_data;
950 0 : a->p_format = j->p_format;
951 0 : a->p_description = j->p_description;
952 :
953 0 : if (j->p_enumerators)
954 0 : a->p_enumerators = new std::vector<std::string>(*j->p_enumerators);
955 :
956 0 : c->p_attributes->push_back(a);
957 : }
958 : }
959 :
960 : // copy direct relationships
961 0 : if (const std::list<OksRelationship*>* rels = src_c.p_relationships) {
962 0 : c->p_relationships = new std::list<OksRelationship*>();
963 :
964 0 : for (const auto& j : *rels) {
965 0 : OksRelationship* r = new OksRelationship(j->p_name, c);
966 :
967 0 : r->p_rclass = j->p_rclass;
968 0 : r->p_low_cc = j->p_low_cc;
969 0 : r->p_high_cc = j->p_high_cc;
970 0 : r->p_composite = j->p_composite;
971 0 : r->p_exclusive = j->p_exclusive;
972 0 : r->p_dependent = j->p_dependent;
973 0 : r->p_description = j->p_description;
974 :
975 0 : c->p_relationships->push_back(r);
976 : }
977 : }
978 :
979 : // copy direct methods
980 0 : if (const std::list<OksMethod*>* mets = src_c.p_methods) {
981 0 : c->p_methods = new std::list<OksMethod*>();
982 :
983 0 : for (const auto& j : *mets) {
984 0 : OksMethod* m = new OksMethod(j->p_name, j->p_description, c);
985 :
986 0 : if (const std::list<OksMethodImplementation*>* impls = j->p_implementations) {
987 0 : m->p_implementations = new std::list<OksMethodImplementation*>();
988 :
989 0 : for (const auto& x : *impls)
990 0 : m->p_implementations->push_back(
991 0 : new OksMethodImplementation(x->get_language(), x->get_prototype(), x->get_body(), m));
992 : }
993 :
994 0 : c->p_methods->push_back(m);
995 : }
996 : }
997 : }
998 :
999 : // copy classes: second iteration to resolve pointers to classes
1000 :
1001 0 : for (const auto& i : src.p_classes) {
1002 0 : OksClass* c = c_table[i.second->p_id];
1003 :
1004 : // link p_class_type of relationship
1005 0 : if (const std::list<OksRelationship*>* rels = i.second->p_relationships) {
1006 0 : for (const auto& j : *rels)
1007 0 : c->find_relationship(j->get_name())->p_class_type =
1008 0 : (j->p_class_type ? c_table[j->p_class_type->p_id] : nullptr);
1009 : }
1010 :
1011 : // copy all super- and sub- classes
1012 0 : if (const OksClass::FList* spcls = i.second->p_all_super_classes) {
1013 0 : c->p_all_super_classes = new OksClass::FList();
1014 :
1015 0 : for (const auto& j : *spcls)
1016 0 : c->p_all_super_classes->push_back(c_table[j->p_id]);
1017 : }
1018 :
1019 0 : if (const OksClass::FList* sbcls = i.second->p_all_sub_classes) {
1020 0 : c->p_all_sub_classes = new OksClass::FList();
1021 :
1022 0 : for (const auto& j : *sbcls)
1023 0 : c->p_all_sub_classes->push_back(c_table[j->p_id]);
1024 : }
1025 :
1026 : // create oks object layout information
1027 0 : c->p_data_info = new OksDataInfo::Map();
1028 0 : size_t instance_size = 0;
1029 :
1030 : // copy all attributes
1031 0 : if (const std::list<OksAttribute*>* attrs = i.second->p_all_attributes) {
1032 0 : c->p_all_attributes = new std::list<OksAttribute*>();
1033 :
1034 0 : for (const auto& j : *attrs) {
1035 0 : OksAttribute* a = (c_table[j->p_class->p_id])->find_direct_attribute(j->get_name());
1036 0 : c->p_all_attributes->push_back(a);
1037 0 : (*c->p_data_info)[a->get_name()] = new OksDataInfo(instance_size++, a);
1038 : }
1039 : }
1040 :
1041 : // copy all relationships
1042 0 : if (const std::list<OksRelationship*>* rels = i.second->p_all_relationships) {
1043 0 : c->p_all_relationships = new std::list<OksRelationship*>();
1044 :
1045 0 : for (const auto& j : *rels) {
1046 0 : OksRelationship* r = (c_table[j->p_class->p_id])->find_direct_relationship(j->get_name());
1047 0 : c->p_all_relationships->push_back(r);
1048 0 : (*c->p_data_info)[r->get_name()] = new OksDataInfo(instance_size++, r);
1049 : }
1050 : }
1051 :
1052 : // copy all methods
1053 0 : if (const std::list<OksMethod*>* mets = i.second->p_all_methods) {
1054 0 : c->p_all_methods = new std::list<OksMethod*>();
1055 :
1056 0 : for (const auto& j : *mets)
1057 0 : c->p_all_methods->push_back((c_table[j->p_class->p_id])->find_direct_method(j->get_name()));
1058 : }
1059 :
1060 : // copy inheritance hierarchy
1061 0 : if (const std::vector<OksClass*>* ih = i.second->p_inheritance_hierarchy) {
1062 0 : c->p_inheritance_hierarchy = new std::vector<OksClass*>();
1063 0 : c->p_inheritance_hierarchy->reserve(ih->size());
1064 :
1065 0 : for (const auto& j : *ih)
1066 0 : c->p_inheritance_hierarchy->push_back(c_table[j->p_id]);
1067 : }
1068 : }
1069 :
1070 : // copy objects
1071 :
1072 0 : if (!src.p_objects.empty()) {
1073 :
1074 : // search an object in a class is not efficien, if there are many such operations
1075 : // use array with class index for fast search
1076 :
1077 0 : OksObject** o_table = new OksObject*[src.p_objects.size()];
1078 0 : idx = 0;
1079 :
1080 : // first iteration: create objects with attributes
1081 :
1082 0 : for (OksObject::Set::const_iterator i = src.p_objects.begin(); i != src.p_objects.end(); ++i, ++idx) {
1083 0 : OksObject* src_o(*i);
1084 0 : src_o->p_user_data = reinterpret_cast<void*>(idx);
1085 :
1086 0 : OksClass* c = c_table[src_o->uid.class_id->p_id];
1087 :
1088 0 : OksObject* o = new OksObject(c,
1089 0 : src_o->uid.object_id,
1090 : src_o->p_user_data,
1091 : src_o->p_int32_id,
1092 : src_o->p_duplicated_object_id_idx,
1093 0 : (*p_data_files.find(&src_o->file->p_full_name)).second);
1094 :
1095 0 : o_table[idx] = o;
1096 0 : (*c->p_objects)[&o->uid.object_id] = o;
1097 0 : p_objects.insert(o);
1098 :
1099 0 : if (size_t num_of_attrs = c->number_of_all_attributes()) {
1100 0 : const OksData* src_data = src_o->data;
1101 0 : OksData* dst_data = o->data;
1102 :
1103 0 : std::list<OksAttribute*>::const_iterator ia_dst = c->all_attributes()->begin();
1104 0 : std::list<OksAttribute*>::const_iterator ia_src = src_o->uid.class_id->all_attributes()->begin();
1105 :
1106 0 : for (size_t j = 0; j < num_of_attrs; ++j) {
1107 0 : *dst_data = *src_data;
1108 :
1109 0 : OksAttribute* a_dst(*ia_dst);
1110 :
1111 : // process in a special way CLASS type (reference on class)
1112 :
1113 0 : if (a_dst->get_data_type() == OksData::class_type) {
1114 0 : if (a_dst->get_is_multi_values() == false) {
1115 0 : dst_data->data.CLASS = c_table[src_data->data.CLASS->p_id];
1116 : } else {
1117 0 : OksData::List::iterator li_dst = dst_data->data.LIST->begin();
1118 0 : OksData::List::iterator li_src = src_data->data.LIST->begin();
1119 :
1120 0 : while (li_dst != dst_data->data.LIST->end()) {
1121 0 : (*li_dst)->data.CLASS = c_table[(*li_src)->data.CLASS->p_id];
1122 0 : ++li_dst;
1123 0 : ++li_src;
1124 : }
1125 : }
1126 : }
1127 :
1128 : // process in a special way ENUM type (reference on attribute's data)
1129 :
1130 0 : else if (a_dst->get_data_type() == OksData::enum_type) {
1131 0 : OksAttribute* a_src(*ia_src);
1132 0 : const std::string* p_enumerators_first(&((*(a_src->p_enumerators))[0]));
1133 :
1134 0 : if (a_dst->get_is_multi_values() == false) {
1135 0 : unsigned long dx = src_data->data.ENUMERATION - p_enumerators_first;
1136 0 : dst_data->data.ENUMERATION = &((*(a_dst->p_enumerators))[dx]);
1137 : } else {
1138 0 : OksData::List::iterator li_dst = dst_data->data.LIST->begin();
1139 0 : OksData::List::iterator li_src = src_data->data.LIST->begin();
1140 :
1141 0 : while (li_dst != dst_data->data.LIST->end()) {
1142 0 : unsigned long dx = (*li_src)->data.ENUMERATION - p_enumerators_first;
1143 0 : (*li_dst)->data.ENUMERATION = &((*(a_dst->p_enumerators))[dx]);
1144 0 : ++li_dst;
1145 0 : ++li_src;
1146 : }
1147 : }
1148 : }
1149 :
1150 0 : src_data++;
1151 0 : dst_data++;
1152 :
1153 0 : ia_dst++;
1154 0 : ia_src++;
1155 : }
1156 : }
1157 : }
1158 :
1159 : // second iteration: link relationships of objects
1160 :
1161 0 : for (const auto& src_o : src.p_objects) {
1162 0 : OksClass* c = c_table[src_o->uid.class_id->p_id];
1163 0 : OksObject* o(o_table[reinterpret_cast<unsigned long>(src_o->p_user_data)]);
1164 :
1165 0 : if (size_t num_of_rels = c->number_of_all_relationships()) {
1166 0 : const OksData* src_data(src_o->data + c->number_of_all_attributes());
1167 0 : OksData* dst_data(o->data + c->number_of_all_attributes());
1168 :
1169 0 : for (size_t j = 0; j < num_of_rels; ++j) {
1170 0 : dst_data->type = src_data->type;
1171 0 : switch (src_data->type) {
1172 0 : case OksData::list_type:
1173 0 : dst_data->data.LIST = new OksData::List();
1174 0 : for (const auto& x : *src_data->data.LIST) {
1175 0 : OksData* d = new OksData();
1176 :
1177 0 : if (x->type == OksData::object_type) {
1178 0 : if (const OksObject* o2 = x->data.OBJECT) {
1179 0 : d->Set(o_table[reinterpret_cast<unsigned long>(o2->p_user_data)]);
1180 : } else {
1181 0 : d->Set((OksObject*)nullptr);
1182 : }
1183 0 : } else if (x->type == OksData::uid_type) {
1184 0 : d->Set(c_table[x->data.UID.class_id->p_id], *x->data.UID.object_id);
1185 0 : } else if (x->type == OksData::uid2_type) {
1186 0 : d->Set(*x->data.UID2.class_id, *x->data.UID2.object_id);
1187 : } else {
1188 0 : ers::error(kernel::InternalError(ERS_HERE, "unexpected data type in relationship list"));
1189 : }
1190 :
1191 0 : dst_data->data.LIST->push_back(d);
1192 : }
1193 0 : break;
1194 :
1195 0 : case OksData::object_type:
1196 0 : if (const OksObject* o2 = src_data->data.OBJECT) {
1197 0 : dst_data->data.OBJECT = o_table[reinterpret_cast<unsigned long>(o2->p_user_data)];
1198 : } else {
1199 0 : dst_data->data.OBJECT = 0;
1200 : }
1201 : break;
1202 :
1203 0 : case OksData::uid_type:
1204 0 : dst_data->data.UID.class_id = c_table[src_data->data.UID.class_id->p_id];
1205 0 : dst_data->data.UID.object_id = new OksString(*src_data->data.UID.object_id);
1206 : break;
1207 :
1208 0 : case OksData::uid2_type:
1209 0 : dst_data->data.UID2.class_id = new OksString(*src_data->data.UID2.class_id);
1210 0 : dst_data->data.UID2.object_id = new OksString(*src_data->data.UID2.object_id);
1211 : break;
1212 :
1213 0 : default:
1214 0 : ers::error(kernel::InternalError(ERS_HERE, "unexpected data type in relationship"));
1215 0 : break;
1216 : }
1217 :
1218 0 : src_data++;
1219 0 : dst_data++;
1220 : }
1221 : }
1222 :
1223 0 : if (const std::list<OksRCR*>* src_rcrs = src_o->p_rcr) {
1224 0 : o->p_rcr = new std::list<OksRCR*>();
1225 :
1226 0 : for (const auto& j : *src_rcrs)
1227 0 : o->p_rcr->push_back(new OksRCR(
1228 0 : o_table[reinterpret_cast<unsigned long>(j->obj->p_user_data)],
1229 0 : (c_table[j->relationship->p_class->p_id])->find_direct_relationship(j->relationship->get_name())));
1230 : } else {
1231 0 : o->p_rcr = nullptr;
1232 : }
1233 : }
1234 :
1235 0 : delete[] o_table;
1236 : }
1237 :
1238 0 : delete[] c_table;
1239 :
1240 : // rename all files
1241 0 : if (copy_repository && p_user_repository_root_inited) {
1242 0 : OksFile::Map* files_map[2] = { &p_schema_files, &p_data_files };
1243 :
1244 0 : for (int i = 0; i < 2; ++i) {
1245 0 : std::vector<OksFile*> files;
1246 :
1247 0 : for (auto& f : *files_map[i])
1248 0 : files.push_back(f.second);
1249 :
1250 0 : files_map[i]->clear();
1251 :
1252 0 : for (auto& f : files) {
1253 0 : std::string name = p_user_repository_root;
1254 0 : name.push_back('/');
1255 0 : name.append(f->get_repository_name());
1256 0 : f->rename(name);
1257 0 : (*files_map[i])[&f->p_full_name] = f;
1258 0 : }
1259 0 : }
1260 : }
1261 0 : }
1262 :
1263 : std::string
1264 7030 : OksKernel::insert_repository_dir(const std::string& dir, bool push_back)
1265 : {
1266 7030 : std::unique_lock lock(p_kernel_mutex);
1267 :
1268 7030 : std::string s(dir);
1269 7030 : Oks::substitute_variables(s);
1270 :
1271 7030 : try {
1272 7030 : Oks::real_path(s, false);
1273 :
1274 6745 : if (std::find(p_repository_dirs.begin(), p_repository_dirs.end(), s) == p_repository_dirs.end()) {
1275 6745 : if (push_back)
1276 6745 : p_repository_dirs.push_back(s);
1277 : else
1278 0 : p_repository_dirs.push_front(s);
1279 6745 : if (p_verbose) {
1280 0 : std::cout << " * push " << (push_back ? "back" : "front") << " repository search directory \'" << s << "\'\n";
1281 : }
1282 6745 : return s;
1283 : }
1284 285 : } catch (exception& ex) {
1285 285 : TLOG_DEBUG(1) << "Cannot insert repository dir \'" << dir << "\':\n\tcaused by: " << ex;
1286 285 : }
1287 :
1288 285 : return "";
1289 7030 : }
1290 :
1291 : void
1292 0 : OksKernel::remove_repository_dir(const std::string& dir)
1293 : {
1294 0 : std::unique_lock lock(p_kernel_mutex);
1295 :
1296 0 : std::list<std::string>::iterator i = std::find(p_repository_dirs.begin(), p_repository_dirs.end(), dir);
1297 0 : if (i != p_repository_dirs.end()) {
1298 0 : if (p_verbose) {
1299 0 : std::cout << " * remove repository search directory \'" << dir << "\'\n";
1300 : }
1301 0 : p_repository_dirs.erase(i);
1302 : }
1303 0 : }
1304 :
1305 : void
1306 0 : OksKernel::set_profiling_mode(const bool b)
1307 : {
1308 0 : if (p_profiling == b)
1309 : return;
1310 :
1311 0 : p_profiling = b;
1312 :
1313 : #ifndef ERS_NO_DEBUG
1314 0 : if (p_profiling == true) {
1315 0 : if (!profiler)
1316 0 : profiler = new OksProfiler();
1317 : } else {
1318 0 : if (profiler) {
1319 0 : delete profiler;
1320 0 : profiler = 0;
1321 : }
1322 : }
1323 : #endif
1324 : }
1325 :
1326 95 : OksKernel::~OksKernel()
1327 : {
1328 95 : {
1329 95 : OSK_PROFILING(OksProfiler::KernelDestructor, this)
1330 95 : OSK_VERBOSE_REPORT("ENTER OksKernel::~OksKernel()")
1331 :
1332 95 : close_all_data();
1333 95 : close_all_schema();
1334 :
1335 95 : std::unique_lock lock(p_kernel_mutex);
1336 :
1337 95 : p_classes.erase(p_classes.begin(), p_classes.end());
1338 :
1339 95 : --p_count;
1340 :
1341 95 : if (p_count == 0) {
1342 95 : std::lock_guard scoped_lock(s_get_cwd_mutex);
1343 95 : delete[] s_cwd;
1344 95 : s_cwd = nullptr;
1345 95 : }
1346 :
1347 95 : remove_user_repository_dir();
1348 95 : }
1349 :
1350 : #ifndef ERS_NO_DEBUG
1351 95 : if (p_profiling)
1352 0 : std::cout << *profiler << std::endl;
1353 : #endif
1354 :
1355 95 : OSK_VERBOSE_REPORT("LEAVE OksKernel::~OksKernel()")
1356 95 : }
1357 :
1358 : std::ostream&
1359 0 : operator<<(std::ostream& s, OksKernel& k)
1360 : {
1361 0 : OSK_PROFILING(OksProfiler::KernelOperatorOut, &k)
1362 :
1363 0 : s << "OKS KERNEL DUMP:\n" << "OKS VERSION: " << k.GetVersion() << std::endl;
1364 :
1365 0 : if (!k.p_schema_files.empty()) {
1366 0 : s << " Loaded schema:\n";
1367 :
1368 0 : for (const auto& i : k.p_schema_files)
1369 0 : s << " " << i.second->get_full_file_name() << std::endl;
1370 :
1371 0 : if (!k.p_data_files.empty()) {
1372 0 : s << " Loaded data:\n";
1373 :
1374 0 : for (const auto& j : k.p_data_files)
1375 0 : s << " " << j.second->get_full_file_name() << std::endl;
1376 : } else
1377 0 : s << " No loaded data files\n";
1378 :
1379 0 : if (!k.p_classes.empty()) {
1380 0 : s << " The classes:\n";
1381 :
1382 0 : for (const auto& j : k.p_classes)
1383 0 : s << *j.second;
1384 : }
1385 : } else
1386 0 : s << " No loaded schema files\n";
1387 :
1388 0 : s << "END OF OKS KERNEL DUMP.\n";
1389 :
1390 0 : return s;
1391 0 : }
1392 :
1393 : bool
1394 336 : OksKernel::check_read_only(OksFile* fp)
1395 : {
1396 355 : static std::string suffix;
1397 336 : static std::once_flag flag;
1398 :
1399 336 : std::call_once(flag, []() {
1400 19 : std::ostringstream s;
1401 19 : s << '-' << get_user_name() << ':' << get_host_name() << ':' << getpid() << std::ends;
1402 19 : suffix = s.str();
1403 19 : });
1404 :
1405 336 : std::string s(fp->p_full_name);
1406 336 : s.append(suffix);
1407 :
1408 336 : {
1409 336 : static std::mutex p_check_read_only_mutex;
1410 336 : std::lock_guard scoped_lock(p_check_read_only_mutex);
1411 :
1412 336 : {
1413 336 : std::ofstream f(s.c_str(), std::ios::out);
1414 336 : fp->p_is_read_only = (!f.good());
1415 336 : }
1416 :
1417 336 : TLOG_DEBUG(3) << "read-only test on file \"" << s << "\" returns " << fp->p_is_read_only;
1418 :
1419 336 : if (!fp->p_is_read_only) {
1420 336 : unlink(s.c_str());
1421 : }
1422 :
1423 336 : return fp->p_is_read_only;
1424 336 : }
1425 336 : }
1426 :
1427 : OksFile*
1428 0 : OksKernel::create_file_info(const std::string& short_path, const std::string& full_path)
1429 : {
1430 0 : OksFile* file_h = 0;
1431 :
1432 0 : {
1433 0 : std::shared_ptr<std::ifstream> f(new std::ifstream(full_path.c_str()));
1434 :
1435 0 : if (f->good()) {
1436 0 : std::shared_ptr<OksXmlInputStream> xmls(new OksXmlInputStream(f));
1437 0 : file_h = new OksFile(xmls, short_path, full_path, this);
1438 0 : }
1439 0 : }
1440 :
1441 : // check file access permissions
1442 :
1443 0 : if (file_h) {
1444 0 : check_read_only(file_h);
1445 : }
1446 :
1447 0 : return file_h;
1448 : }
1449 :
1450 : // The function tests file existence, touches the file,
1451 : // prints out warnings/errors in case of problems
1452 : // and finally prints out info message.
1453 :
1454 : static void
1455 5 : test_file_existence(const std::string& file_name, bool silence, const std::string& fname, const char* msg)
1456 : {
1457 : // test file existence
1458 :
1459 5 : bool file_exists = false;
1460 :
1461 5 : {
1462 5 : struct stat buf;
1463 5 : if (stat(file_name.c_str(), &buf) == 0) {
1464 4 : if (!silence) {
1465 0 : Oks::warning_msg(fname) << " File \"" << file_name << "\" already exists\n";
1466 : }
1467 : file_exists = true;
1468 : }
1469 : }
1470 :
1471 : // test file permissions
1472 :
1473 5 : {
1474 5 : std::ofstream f(file_name.c_str());
1475 :
1476 5 : if (!f) {
1477 0 : if (!silence) {
1478 0 : if (file_exists) {
1479 0 : throw std::runtime_error("cannot open file in write mode");
1480 : } else {
1481 0 : throw std::runtime_error("cannot create file");
1482 : }
1483 : }
1484 : }
1485 5 : }
1486 :
1487 5 : if (!silence)
1488 0 : std::cout << "Creating new " << msg << " file \"" << file_name << "\"..." << std::endl;
1489 5 : }
1490 :
1491 : inline std::string
1492 13803 : mk_name_and_test(const std::string& name, const char* test, size_t test_len)
1493 : {
1494 13803 : const char _s1_str[] = " - \'";
1495 13803 : const char _s2_str[] = "\' tested as ";
1496 :
1497 27606 : return ((std::string(_s1_str, sizeof(_s1_str) - 1) + name).append(_s2_str, sizeof(_s2_str) - 1))
1498 27606 : .append(test, test_len);
1499 : }
1500 :
1501 : #define TEST_PATH_TOKEN(path, file, msg) \
1502 : std::string token(path); \
1503 : token.push_back('/'); \
1504 : token.append(file); \
1505 : Oks::substitute_variables(token); \
1506 : if (Oks::real_path(token, true)) { \
1507 : TLOG_DEBUG(2) << fname << " returns \'" << token << "\' (filename relative to " << msg \
1508 : << " database repository directory)"; \
1509 : return token; \
1510 : } else { \
1511 : const char _test_name[] = "relative to database repository directory"; \
1512 : tested_files.push_back(mk_name_and_test(token, _test_name, sizeof(_test_name) - 1)); \
1513 : }
1514 :
1515 : std::string
1516 504 : OksKernel::get_file_path(const std::string& s, const OksFile* file_h, bool strict_paths) const
1517 : {
1518 504 : strict_paths &= p_use_strict_repository_paths;
1519 :
1520 504 : const char _fname[] = "get_file_path";
1521 504 : std::string fname;
1522 :
1523 504 : TLOG_DEBUG(2) << "ENTER " << make_fname(_fname, sizeof(_fname) - 1, s, nullptr, &file_h);
1524 :
1525 504 : std::list<std::string> tested_files;
1526 :
1527 : // check absolute path or path relative to working directory
1528 504 : bool is_absolute_path = false;
1529 :
1530 504 : std::string s2(s);
1531 504 : Oks::substitute_variables(s2);
1532 504 : fname = make_fname(_fname, sizeof(_fname) - 1, s2, nullptr, &file_h);
1533 :
1534 : // test if file has an absolute path
1535 504 : if (s2[0] == '/')
1536 : is_absolute_path = true;
1537 :
1538 : // continue only if the file is an absolute path or it is not included
1539 448 : if (is_absolute_path || file_h == 0) {
1540 81 : if (!is_absolute_path && s_cwd && *s_cwd) {
1541 81 : std::string s3 = s_cwd;
1542 81 : s3.push_back('/');
1543 81 : s2 = s3 + s2;
1544 81 : }
1545 :
1546 137 : if (Oks::real_path(s2, true)) {
1547 56 : if (!get_user_repository_root().empty() && strict_paths) {
1548 0 : if (!get_repository_mapping_dir().empty() && s2.find(get_repository_mapping_dir()) == 0) {
1549 0 : s2.erase(0, get_repository_mapping_dir().size() + 1);
1550 0 : TEST_PATH_TOKEN(get_user_repository_root(), s2, "user")
1551 0 : } else if (s2.find(get_user_repository_root()) == 0) {
1552 : // presumably from explicitly set TDAQ_DB_USER_REPOSITORY created externally
1553 0 : TLOG_DEBUG(2) << fname << " returns external \'" << s2 << "\' (an absolute filename or relative to CWD=\'"
1554 0 : << s_cwd << "\')";
1555 0 : return s2;
1556 : }
1557 :
1558 0 : std::ostringstream text;
1559 0 : text << fname
1560 : << " file does not belong to oks git repository\n"
1561 : "provide file repository name, or unset TDAQ_DB_REPOSITORY and try again with the DUNEDAQ_DB_PATH "
1562 0 : "environment variable";
1563 0 : throw std::runtime_error(text.str().c_str());
1564 0 : } else {
1565 56 : TLOG_DEBUG(2) << fname << " returns \'" << s2 << "\' (an absolute filename or relative to CWD=\'" << s_cwd
1566 56 : << "\')";
1567 56 : return s2;
1568 : }
1569 : } else {
1570 81 : if (is_absolute_path) {
1571 0 : const char _test_name[] = "an absolute file name";
1572 0 : tested_files.push_back(mk_name_and_test(s2, _test_name, sizeof(_test_name) - 1));
1573 : } else {
1574 81 : const char _test_name[] = "relative to current working directory";
1575 81 : tested_files.push_back(mk_name_and_test(s2, _test_name, sizeof(_test_name) - 1));
1576 : }
1577 : }
1578 : }
1579 :
1580 448 : if (!get_user_repository_root().empty()) {
1581 0 : TEST_PATH_TOKEN(get_user_repository_root(), s, "user")
1582 0 : }
1583 :
1584 448 : if (get_user_repository_root().empty() || !p_use_strict_repository_paths) {
1585 : // check paths relative to OKS database root directories
1586 448 : if (!is_absolute_path)
1587 14170 : for (auto& i : p_repository_dirs) {
1588 14170 : TEST_PATH_TOKEN(i, s, "DUNEDAQ_DB_PATH")
1589 14170 : }
1590 :
1591 : // check non-absolute path relative to parent file if any
1592 0 : if (file_h && !is_absolute_path) {
1593 0 : std::string s2(file_h->get_full_file_name());
1594 0 : std::string::size_type pos = s2.find_last_of('/');
1595 :
1596 0 : if (pos != std::string::npos) {
1597 0 : s2.erase(pos + 1);
1598 0 : s2.append(s);
1599 0 : Oks::substitute_variables(s2);
1600 :
1601 0 : if (Oks::real_path(s2, true)) {
1602 0 : TLOG_DEBUG(2) << fname << " returns \'" << s2 << "\' (filename relative to parent file)";
1603 0 : return s2;
1604 : } else {
1605 0 : const char _test_name[] = "relative to parent file";
1606 0 : tested_files.push_back(mk_name_and_test(s2, _test_name, sizeof(_test_name) - 1));
1607 : }
1608 : }
1609 0 : }
1610 : }
1611 :
1612 0 : TLOG_DEBUG(2) << fname << " throw exception (file was not found)";
1613 :
1614 0 : std::ostringstream text;
1615 0 : text << fname << " found no readable file among " << tested_files.size() << " tested:\n";
1616 0 : for (auto& i : tested_files)
1617 0 : text << i << std::endl;
1618 :
1619 0 : throw std::runtime_error(text.str().c_str());
1620 504 : }
1621 :
1622 : // user-allowed method
1623 :
1624 : OksFile*
1625 90 : OksKernel::load_file(const std::string& short_file_name, bool bind)
1626 : {
1627 90 : std::unique_lock lock(p_kernel_mutex);
1628 180 : return k_load_file(short_file_name, bind, 0, 0);
1629 90 : }
1630 :
1631 : // kernel method
1632 :
1633 : OksFile*
1634 457 : OksKernel::k_load_file(const std::string& short_file_name, bool bind, const OksFile* parent_h, OksPipeline* pipeline)
1635 : {
1636 457 : const char _fname[] = "k_load_file";
1637 457 : std::string fname = make_fname(_fname, sizeof(_fname) - 1, short_file_name, &bind, &parent_h);
1638 :
1639 457 : OSK_VERBOSE_REPORT("ENTER " << fname)
1640 :
1641 457 : std::string full_file_name;
1642 :
1643 457 : try {
1644 457 : full_file_name = get_file_path(short_file_name, parent_h);
1645 0 : } catch (std::exception& e) {
1646 0 : throw CanNotOpenFile("k_load_file", short_file_name, e.what());
1647 0 : }
1648 :
1649 457 : try {
1650 :
1651 : // check if the file is already loaded
1652 :
1653 457 : OksFile::Map::const_iterator i = p_schema_files.find(&full_file_name);
1654 457 : if (i != p_schema_files.end())
1655 142 : return i->second->check_parent(parent_h);
1656 :
1657 315 : i = p_data_files.find(&full_file_name);
1658 315 : if (i != p_data_files.end())
1659 0 : return i->second->check_parent(parent_h);
1660 :
1661 315 : std::shared_ptr<std::ifstream> f(new std::ifstream(full_file_name.c_str()));
1662 :
1663 315 : if (f->good()) {
1664 315 : long file_length(get_file_length(*f));
1665 :
1666 315 : if (!file_length) {
1667 0 : throw std::runtime_error("k_load_file(): file is empty");
1668 : }
1669 :
1670 : // read file header and decide what to load
1671 :
1672 315 : std::shared_ptr<OksXmlInputStream> xmls(new OksXmlInputStream(f));
1673 :
1674 315 : OksFile* file_h = new OksFile(xmls, short_file_name, full_file_name, this);
1675 :
1676 315 : if (file_h->p_oks_format.size() == 6 && cmp_str6n(file_h->p_oks_format.c_str(), "schema")) {
1677 182 : k_load_schema(file_h, xmls, parent_h);
1678 : } else {
1679 133 : char format;
1680 :
1681 133 : if (file_h->p_oks_format.size() == 4 && cmp_str4n(file_h->p_oks_format.c_str(), "data"))
1682 : format = 'n';
1683 0 : else if (file_h->p_oks_format.size() == 8 && cmp_str8n(file_h->p_oks_format.c_str(), "extended"))
1684 : format = 'X';
1685 0 : else if (file_h->p_oks_format.size() == 7 && cmp_str7n(file_h->p_oks_format.c_str(), "compact"))
1686 : format = 'c';
1687 : else {
1688 0 : delete file_h;
1689 0 : throw std::runtime_error("k_load_file(): failed to parse header");
1690 : }
1691 :
1692 133 : k_load_data(file_h, format, xmls, file_length, bind, parent_h, pipeline);
1693 : }
1694 :
1695 315 : OSK_VERBOSE_REPORT("LEAVE " << fname)
1696 :
1697 315 : return file_h;
1698 315 : } else {
1699 0 : throw std::runtime_error("k_load_file(): cannot open file");
1700 : }
1701 315 : } catch (exception& e) {
1702 0 : throw FailedLoadFile("file", full_file_name, e);
1703 0 : } catch (std::exception& e) {
1704 0 : throw FailedLoadFile("file", full_file_name, e.what());
1705 0 : } catch (...) {
1706 0 : throw FailedLoadFile("file", full_file_name, "caught unknown exception");
1707 0 : }
1708 457 : }
1709 :
1710 : void
1711 315 : OksKernel::k_load_includes(const OksFile& f, OksPipeline* pipeline)
1712 : {
1713 315 : for (std::list<std::string>::const_iterator i = f.p_list_of_include_files.begin();
1714 672 : i != f.p_list_of_include_files.end();
1715 357 : ++i) {
1716 357 : if (!(*i).empty()) {
1717 : // if(f.get_repository() != OksFile::NoneRepository) {
1718 357 : if (!get_user_repository_root().empty()) {
1719 0 : if ((*i)[0] == '/') {
1720 0 : throw FailedLoadFile("include",
1721 0 : *i,
1722 : "inclusion of files with absolute pathname (like \"/foo/bar\") is not allowed by "
1723 0 : "repository files; include files relative to repository root (like \"foo/bar\")");
1724 0 : } else if ((*i)[0] == '.') {
1725 0 : throw FailedLoadFile(
1726 : "include",
1727 0 : *i,
1728 : "inclusion of files with file-related relative pathnames (like \"../foo/bar\" or \"./bar\") is not "
1729 0 : "supported by repository files; include files relative to repository root (like \"foo/bar\")");
1730 0 : } else if ((*i).find('/') == std::string::npos) {
1731 0 : throw FailedLoadFile("include",
1732 0 : *i,
1733 : "inclusion of files with file-related local pathnames (like \"bar\") is not supported "
1734 0 : "by repository files; include files relative to repository root (like \"foo/bar\")");
1735 : }
1736 : }
1737 :
1738 357 : try {
1739 357 : k_load_file(*i, false, &f, pipeline);
1740 0 : } catch (exception& e) {
1741 0 : throw FailedLoadFile("include", *i, e);
1742 0 : } catch (std::exception& e) {
1743 0 : throw FailedLoadFile("include", *i, e.what());
1744 0 : }
1745 : } else {
1746 0 : throw FailedLoadFile("include", *i, "empty filename");
1747 : }
1748 : }
1749 315 : }
1750 :
1751 : void
1752 0 : OksKernel::get_includes(const std::string& file_name, std::set<std::string>& includes, bool use_repository_name)
1753 : {
1754 0 : try {
1755 0 : std::shared_ptr<std::ifstream> f(new std::ifstream(file_name.c_str()));
1756 :
1757 0 : if (!f->good()) {
1758 0 : throw std::runtime_error("get_includes(): cannot open file");
1759 : }
1760 :
1761 0 : std::shared_ptr<OksXmlInputStream> xmls(new OksXmlInputStream(f));
1762 0 : OksFile fp(xmls, file_name, file_name, this);
1763 :
1764 0 : if (fp.p_list_of_include_files.size()) {
1765 0 : for (std::list<std::string>::iterator i = fp.p_list_of_include_files.begin();
1766 0 : i != fp.p_list_of_include_files.end();
1767 0 : ++i) {
1768 0 : includes.insert(use_repository_name ? get_file_path(*i, &fp).substr(get_user_repository_root().size() + 1)
1769 0 : : get_file_path(*i, &fp));
1770 : }
1771 : }
1772 :
1773 0 : f->close();
1774 0 : } catch (exception& e) {
1775 0 : throw(FailedLoadFile("file", file_name, e));
1776 0 : } catch (std::exception& e) {
1777 0 : throw(FailedLoadFile("file", file_name, e.what()));
1778 0 : }
1779 0 : }
1780 :
1781 : bool
1782 0 : OksKernel::test_parent(OksFile* file, OksFile::IMap::iterator& i)
1783 : {
1784 0 : OksFile* parent(i->first);
1785 0 : OksFile::Set& includes(i->second);
1786 :
1787 0 : if (includes.find(file) != includes.end()) {
1788 0 : if (file->p_included_by != parent) {
1789 0 : TLOG_DEBUG(1) << "new parent of file " << file->get_full_file_name() << " is " << parent->get_full_file_name();
1790 0 : file->p_included_by = parent;
1791 : }
1792 :
1793 0 : return true;
1794 : }
1795 :
1796 : return false;
1797 : }
1798 :
1799 : void
1800 0 : OksKernel::k_close_dangling_includes()
1801 : {
1802 : // build inclusion graph
1803 :
1804 0 : OksFile::IMap igraph;
1805 :
1806 0 : for (OksFile::Map::iterator i = p_schema_files.begin(); i != p_schema_files.end(); ++i) {
1807 0 : for (std::list<std::string>::const_iterator x = i->second->p_list_of_include_files.begin();
1808 0 : x != i->second->p_list_of_include_files.end();
1809 0 : ++x) {
1810 0 : std::string f = get_file_path(*x, i->second);
1811 0 : OksFile::Map::const_iterator it = p_schema_files.find(&f);
1812 0 : if (it != p_schema_files.end()) {
1813 0 : igraph[i->second].insert(it->second);
1814 : } else {
1815 0 : std::cerr << "cannot find schema " << *x << " included by " << i->second->get_full_file_name() << std::endl;
1816 : }
1817 0 : }
1818 : }
1819 :
1820 0 : for (OksFile::Map::iterator i = p_data_files.begin(); i != p_data_files.end(); ++i) {
1821 0 : for (std::list<std::string>::const_iterator x = i->second->p_list_of_include_files.begin();
1822 0 : x != i->second->p_list_of_include_files.end();
1823 0 : ++x) {
1824 0 : std::string f = get_file_path(*x, i->second);
1825 0 : OksFile::Map::const_iterator it = p_data_files.find(&f);
1826 0 : if (it != p_data_files.end()) {
1827 0 : igraph[i->second].insert(it->second);
1828 : } else {
1829 0 : it = p_schema_files.find(&f);
1830 :
1831 0 : if (it != p_schema_files.end()) {
1832 0 : igraph[i->second].insert(it->second);
1833 : } else {
1834 0 : std::cerr << "cannot find file " << *x << " included by " << i->second->get_full_file_name() << std::endl;
1835 : }
1836 : }
1837 0 : }
1838 : }
1839 :
1840 0 : const size_t num_of_schema_files(schema_files().size());
1841 :
1842 : // calculate files to be closed
1843 :
1844 0 : while (true) {
1845 0 : std::list<OksFile*> schema_files, data_files; // files to be closed
1846 :
1847 0 : OksFile::Map* files[2] = { &p_schema_files, &p_data_files };
1848 0 : std::list<OksFile*> to_be_closed[2];
1849 :
1850 0 : for (int c = 0; c < 2; ++c) {
1851 0 : for (OksFile::Map::iterator i = files[c]->begin(); i != files[c]->end(); ++i) {
1852 0 : if (OksFile* p = const_cast<OksFile*>(i->second->p_included_by)) {
1853 0 : bool found_parent = false;
1854 :
1855 : // check, the parent is valid
1856 :
1857 0 : if (igraph.find(p) != igraph.end()) {
1858 0 : OksFile::IMap::iterator j = igraph.find(p);
1859 0 : if (test_parent(i->second, j)) {
1860 0 : continue;
1861 : }
1862 : }
1863 :
1864 0 : TLOG_DEBUG(1) << "the parent of file " << i->second->get_full_file_name() << " is not valid";
1865 :
1866 : // try to find new parent
1867 :
1868 0 : for (OksFile::IMap::iterator j = igraph.begin(); j != igraph.end(); ++j) {
1869 0 : if (test_parent(i->second, j)) {
1870 : found_parent = true;
1871 : break;
1872 : }
1873 : }
1874 :
1875 0 : if (found_parent == false) {
1876 0 : TLOG_DEBUG(1) << "the file " << i->second->get_full_file_name()
1877 0 : << " is not included by any other file and will be closed";
1878 0 : to_be_closed[c].push_back(i->second);
1879 : }
1880 : }
1881 : }
1882 : }
1883 :
1884 0 : int num = 0;
1885 :
1886 0 : for (std::list<OksFile*>::const_iterator i = to_be_closed[1].begin(); i != to_be_closed[1].end(); ++i) {
1887 0 : num += (*i)->p_list_of_include_files.size();
1888 0 : k_close_data(*i, true);
1889 0 : igraph.erase(*i);
1890 : }
1891 :
1892 0 : for (std::list<OksFile*>::const_iterator i = to_be_closed[0].begin(); i != to_be_closed[0].end(); ++i) {
1893 0 : num += (*i)->p_list_of_include_files.size();
1894 0 : k_close_schema(*i);
1895 0 : igraph.erase(*i);
1896 : }
1897 :
1898 0 : if (num > 0) {
1899 0 : TLOG_DEBUG(1) << "go into recursive call, number of potentially closed includes is " << num;
1900 : } else {
1901 0 : TLOG_DEBUG(1) << "break loop";
1902 0 : break;
1903 : }
1904 0 : }
1905 :
1906 0 : if (num_of_schema_files != schema_files().size()) {
1907 0 : TLOG_DEBUG(1) << "rebuild classes since number of schema files has been changed from " << num_of_schema_files
1908 0 : << " to " << schema_files().size();
1909 0 : registrate_all_classes();
1910 : }
1911 0 : }
1912 :
1913 : bool
1914 0 : OksKernel::k_preload_includes(OksFile* fp, std::set<OksFile*>& new_files_h, bool allow_schema_extension)
1915 : {
1916 0 : bool found_include_changes(false);
1917 :
1918 0 : try {
1919 0 : std::shared_ptr<std::ifstream> file(new std::ifstream(fp->get_full_file_name().c_str()));
1920 :
1921 0 : if (!file->good()) {
1922 0 : throw std::runtime_error(std::string("k_preload_includes(): cannot open file \"") + fp->get_full_file_name() +
1923 0 : '\"');
1924 : }
1925 :
1926 : // keep included files
1927 :
1928 0 : std::set<std::string> included;
1929 :
1930 0 : if (fp->p_list_of_include_files.size()) {
1931 0 : for (std::list<std::string>::iterator i = fp->p_list_of_include_files.begin();
1932 0 : i != fp->p_list_of_include_files.end();
1933 0 : ++i) {
1934 0 : included.insert(*i);
1935 : }
1936 : }
1937 :
1938 : // get size of file and check that it is not empty
1939 :
1940 0 : file->seekg(0, std::ios::end);
1941 :
1942 0 : long file_length = static_cast<std::streamoff>(file->tellg());
1943 :
1944 0 : if (file_length == 0) {
1945 0 : throw std::runtime_error(std::string("k_preload_includes(): file \"") + fp->get_full_file_name() + "\" is empty");
1946 : }
1947 :
1948 0 : file->seekg(0, std::ios::beg);
1949 :
1950 0 : std::shared_ptr<OksXmlInputStream> xmls(new OksXmlInputStream(file));
1951 :
1952 0 : {
1953 0 : OksFile fp2(xmls, fp->get_short_file_name(), fp->get_full_file_name(), this);
1954 0 : fp2.p_included_by = fp->p_included_by;
1955 0 : p_preload_file_info[fp] = new OksFile(*fp);
1956 0 : *fp = fp2;
1957 :
1958 0 : if (fp->p_oks_format.empty()) {
1959 0 : throw std::runtime_error(std::string("k_preload_includes(): failed to read header of file \"") +
1960 0 : fp->get_full_file_name() + '\"');
1961 0 : } else if (fp->p_oks_format == "schema") {
1962 0 : TLOG_DEBUG(2) << "skip reload of schema file \'" << fp->get_full_file_name() << '\'';
1963 0 : return false;
1964 0 : } else if (fp->p_oks_format != "data" && fp->p_oks_format != "extended" && fp->p_oks_format != "compact") {
1965 0 : throw std::runtime_error(std::string("k_preload_includes(): file \"") + fp->get_full_file_name() +
1966 0 : "\" is not valid oks file");
1967 : }
1968 0 : }
1969 :
1970 0 : if (fp->p_list_of_include_files.size()) {
1971 0 : TLOG_DEBUG(3) << "check \'include\' of the file \'" << fp->get_full_file_name() << '\'';
1972 :
1973 0 : if (included.size() != fp->p_list_of_include_files.size())
1974 0 : found_include_changes = true;
1975 :
1976 0 : for (std::list<std::string>::iterator i = fp->p_list_of_include_files.begin();
1977 0 : i != fp->p_list_of_include_files.end();
1978 0 : ++i) {
1979 0 : std::set<std::string>::const_iterator x = included.find(*i);
1980 0 : if (x != included.end()) {
1981 0 : TLOG_DEBUG(3) << "include \'" << *i << "\' already exists, skip...";
1982 : } else {
1983 0 : TLOG_DEBUG(3) << "the file \'" << *i << "\' was not previously included by \'" << fp->get_full_file_name()
1984 0 : << '\'';
1985 :
1986 0 : found_include_changes = true;
1987 :
1988 0 : std::string full_file_name;
1989 :
1990 0 : try {
1991 0 : full_file_name = get_file_path(*i, fp);
1992 0 : } catch (std::exception& e) {
1993 0 : throw CanNotOpenFile("k_preload_includes", *i, e.what());
1994 0 : }
1995 :
1996 0 : OksFile::Map::const_iterator j = p_schema_files.find(&full_file_name);
1997 0 : if (j != p_schema_files.end()) {
1998 0 : TLOG_DEBUG(3) << "the include \'" << *i << "\' is already loaded schema file \'" << j->first << '\'';
1999 0 : continue;
2000 0 : }
2001 :
2002 0 : j = p_data_files.find(&full_file_name);
2003 0 : if (j != p_data_files.end()) {
2004 0 : TLOG_DEBUG(3) << "the include \'" << *i << "\' is already loaded data file \'" << j->first << '\'';
2005 0 : continue;
2006 0 : }
2007 :
2008 0 : if (OksFile* f = create_file_info(*i, full_file_name)) {
2009 0 : if (f->p_oks_format == "schema") {
2010 0 : std::string new_schema_full_file_name = f->get_full_file_name();
2011 0 : delete f;
2012 0 : if (p_schema_files.find(&new_schema_full_file_name) != p_schema_files.end()) {
2013 0 : TLOG_DEBUG(3) << "the include \'" << *i << "\' is a schema file, that was already loaded";
2014 0 : continue;
2015 0 : } else {
2016 0 : if (allow_schema_extension) {
2017 0 : TLOG_DEBUG(3) << "the include \'" << *i << "\' is new schema file, loading...";
2018 0 : k_load_schema(*i, fp);
2019 0 : continue;
2020 0 : } else {
2021 0 : std::ostringstream text;
2022 0 : text << "k_preload_includes(): include of new schema file (\'" << *i
2023 0 : << "\') is not allowed on data reload";
2024 0 : throw std::runtime_error(text.str().c_str());
2025 0 : }
2026 : }
2027 0 : } else if (f->p_oks_format == "data" || f->p_oks_format == "extended" || f->p_oks_format == "compact") {
2028 0 : TLOG_DEBUG(3) << "the include \'" << *i << "\' is new data file, pre-loading...";
2029 0 : add_data_file(f);
2030 0 : p_preload_added_files.push_back(f);
2031 0 : new_files_h.insert(f);
2032 0 : f->p_list_of_include_files.clear();
2033 0 : if (k_preload_includes(f, new_files_h, allow_schema_extension))
2034 0 : found_include_changes = true;
2035 0 : f->p_included_by = fp;
2036 0 : f->update_status_of_file();
2037 : } else {
2038 0 : delete f;
2039 0 : std::ostringstream text;
2040 0 : text << "k_preload_includes(): failed to parse header of included \'" << full_file_name << "\' file";
2041 0 : throw std::runtime_error(text.str().c_str());
2042 0 : }
2043 : } else {
2044 0 : throw std::runtime_error("k_load_file(): cannot open file");
2045 : }
2046 0 : }
2047 : }
2048 : }
2049 0 : } catch (exception& e) {
2050 0 : throw FailedLoadFile("data file", fp->get_full_file_name(), e);
2051 0 : } catch (std::exception& e) {
2052 0 : throw FailedLoadFile("data file", fp->get_full_file_name(), e.what());
2053 0 : }
2054 :
2055 0 : return found_include_changes;
2056 : }
2057 :
2058 : /******************************************************************************/
2059 :
2060 : std::string
2061 0 : OksKernel::create_user_repository_dir()
2062 : {
2063 0 : if (const char* path = getenv("TDAQ_DB_USER_REPOSITORY_PATH"))
2064 0 : return path;
2065 :
2066 : // create user repository directory if necessary
2067 0 : std::string user_repo;
2068 :
2069 0 : if (const char* user_repo_base = getenv("TDAQ_DB_USER_REPOSITORY_ROOT"))
2070 0 : user_repo = user_repo_base;
2071 : else
2072 0 : user_repo = get_temporary_dir();
2073 :
2074 0 : user_repo.push_back('/');
2075 0 : if (const char* user_repo_pattern = getenv("TDAQ_DB_USER_REPOSITORY_PATTERN"))
2076 0 : user_repo.append(user_repo_pattern);
2077 : else
2078 0 : user_repo.append("oks.XXXXXX");
2079 :
2080 0 : std::unique_ptr<char[]> dir_template(new char[user_repo.size() + 1]);
2081 0 : strcpy(dir_template.get(), user_repo.c_str());
2082 :
2083 0 : if (char* tmp_dirname = mkdtemp(dir_template.get())) {
2084 0 : return tmp_dirname;
2085 : }
2086 :
2087 0 : throw CanNotCreateRepositoryDir("make_user_repository_dir", user_repo);
2088 0 : }
2089 :
2090 : OksFile*
2091 32 : OksKernel::find_file(const std::string& s, const OksFile::Map& files) const
2092 : {
2093 32 : try {
2094 32 : std::string at = get_file_path(s, nullptr);
2095 32 : OksFile::Map::const_iterator i = files.find(&at);
2096 32 : if (i != files.end())
2097 27 : return (*i).second;
2098 32 : } catch (...) {
2099 : // return 0;
2100 0 : }
2101 :
2102 : // search files included as relative to parent
2103 5 : for (const auto& f : files) {
2104 0 : if (s == f.second->get_short_file_name()) {
2105 0 : return f.second;
2106 : }
2107 : }
2108 :
2109 5 : return nullptr;
2110 : }
2111 :
2112 : OksFile*
2113 0 : OksKernel::find_schema_file(const std::string& s) const
2114 : {
2115 0 : return find_file(s, p_schema_files);
2116 : }
2117 :
2118 : OksFile*
2119 32 : OksKernel::find_data_file(const std::string& s) const
2120 : {
2121 32 : return find_file(s, p_data_files);
2122 : }
2123 :
2124 : /******************************************************************************/
2125 :
2126 : // user-allowed method
2127 :
2128 : OksFile*
2129 0 : OksKernel::load_schema(const std::string& short_file_name, const OksFile* parent_h)
2130 : {
2131 0 : std::unique_lock lock(p_kernel_mutex);
2132 0 : return k_load_schema(short_file_name, parent_h);
2133 0 : }
2134 :
2135 : // kernel method
2136 :
2137 : OksFile*
2138 0 : OksKernel::k_load_schema(const std::string& short_file_name, const OksFile* parent_h)
2139 : {
2140 0 : const char _fname[] = "k_load_schema";
2141 0 : std::string fname = make_fname(_fname, sizeof(_fname) - 1, short_file_name, nullptr, &parent_h);
2142 :
2143 0 : OSK_VERBOSE_REPORT("ENTER " << fname)
2144 :
2145 0 : std::string full_file_name;
2146 :
2147 0 : try {
2148 0 : full_file_name = get_file_path(short_file_name, parent_h);
2149 0 : } catch (std::exception& e) {
2150 0 : throw CanNotOpenFile("k_load_schema", short_file_name, e.what());
2151 0 : }
2152 :
2153 0 : try {
2154 0 : OksFile* fp = find_schema_file(full_file_name);
2155 :
2156 0 : if (fp != 0) {
2157 0 : if (p_verbose) {
2158 0 : std::lock_guard lock(p_parallel_out_mutex);
2159 0 : Oks::warning_msg(fname) << " The file was already loaded\n";
2160 0 : }
2161 0 : return fp->check_parent(parent_h);
2162 : }
2163 :
2164 0 : {
2165 0 : std::shared_ptr<std::ifstream> f(new std::ifstream(full_file_name.c_str()));
2166 :
2167 0 : if (!f->good()) {
2168 0 : throw std::runtime_error("k_load_schema(): cannot open file");
2169 : }
2170 :
2171 0 : std::shared_ptr<OksXmlInputStream> xmls(new OksXmlInputStream(f));
2172 :
2173 0 : fp = new OksFile(xmls, short_file_name, full_file_name, this);
2174 :
2175 0 : if (fp->p_oks_format.empty()) {
2176 0 : throw std::runtime_error("k_load_schema(): failed to read header of file");
2177 0 : } else if (fp->p_oks_format != "schema") {
2178 0 : throw std::runtime_error("k_load_schema(): file is not oks schema file");
2179 : }
2180 :
2181 0 : k_load_schema(fp, xmls, parent_h);
2182 :
2183 0 : OSK_VERBOSE_REPORT("LEAVE " << fname)
2184 :
2185 0 : return fp;
2186 0 : }
2187 0 : } catch (FailedLoadFile&) {
2188 0 : throw; // forward
2189 0 : } catch (exception& e) {
2190 0 : throw FailedLoadFile("schema file", full_file_name, e);
2191 0 : } catch (std::exception& e) {
2192 0 : throw FailedLoadFile("schema file", full_file_name, e.what());
2193 0 : } catch (...) {
2194 0 : throw FailedLoadFile("schema file", full_file_name, "caught unknown exception");
2195 0 : }
2196 0 : }
2197 :
2198 : void
2199 182 : OksKernel::k_load_schema(OksFile* fp, std::shared_ptr<OksXmlInputStream> xmls, const OksFile* parent_h)
2200 : {
2201 182 : OSK_PROFILING(OksProfiler::KernelLoadSchema, this)
2202 :
2203 182 : fp->p_included_by = parent_h;
2204 182 : check_read_only(fp);
2205 :
2206 182 : try {
2207 182 : if (!p_silence) {
2208 0 : std::lock_guard lock(p_parallel_out_mutex);
2209 0 : std::cout << (parent_h ? " * loading " : "Loading ") << fp->p_number_of_items << " classes from file \""
2210 0 : << fp->get_full_file_name() << "\"...\n";
2211 0 : if (parent_h == 0 && fp->get_full_file_name() != fp->get_short_file_name()) {
2212 0 : std::cout << "(non-fully-qualified filename was \"" << fp->get_short_file_name() << "\")\n";
2213 : }
2214 0 : }
2215 :
2216 182 : k_load_includes(*fp, 0);
2217 :
2218 182 : add_schema_file(fp);
2219 :
2220 182 : std::list<OksClass*> set;
2221 :
2222 15186 : while (true) {
2223 7684 : OksClass* c = 0;
2224 :
2225 7684 : try {
2226 7684 : c = new OksClass(*xmls, this);
2227 182 : } catch (EndOfXmlStream&) {
2228 182 : delete c;
2229 182 : break;
2230 182 : }
2231 :
2232 7502 : if (c->get_name().empty()) {
2233 0 : throw std::runtime_error("k_load_schema(): failed to read a class");
2234 : }
2235 :
2236 7502 : OksClass::Map::const_iterator ic = p_classes.find(c->get_name().c_str());
2237 7502 : if (ic != p_classes.end()) {
2238 0 : bool are_different = (*ic->second != *c);
2239 :
2240 0 : c->p_transient = true;
2241 0 : delete c;
2242 0 : c = ic->second;
2243 0 : if (p_allow_duplicated_classes && !are_different) {
2244 0 : static bool ers_report = (getenv("OKS_KERNEL_ERS_REPORT_DUPLICATED_CLASSES") != nullptr);
2245 0 : if (ers_report) {
2246 0 : ers::warning(kernel::ClassAlreadyDefined(
2247 0 : ERS_HERE, c->get_name(), fp->get_full_file_name(), c->p_file->get_full_file_name()));
2248 : } else {
2249 0 : const char _fname[] = "k_load_schema";
2250 0 : std::lock_guard lock(p_parallel_out_mutex);
2251 0 : Oks::warning_msg(make_fname(_fname, sizeof(_fname) - 1, fp->get_full_file_name(), nullptr, &parent_h))
2252 0 : << " Class \"" << c->get_name() << "\" was already loaded from file \'"
2253 0 : << c->get_file()->get_full_file_name() << "\'\n";
2254 0 : }
2255 : } else {
2256 0 : std::stringstream text;
2257 0 : text << (are_different ? "different " : "") << "class \"" << c->get_name()
2258 0 : << "\" was already loaded from file \'" << c->get_file()->get_full_file_name() << '\'';
2259 0 : throw std::runtime_error(text.str().c_str());
2260 0 : }
2261 : } else {
2262 7502 : {
2263 7502 : std::unique_lock lock(p_schema_mutex); // protect schema and all objects from changes
2264 7502 : p_classes[c->get_name().c_str()] = c;
2265 7502 : }
2266 7502 : c->p_file = fp;
2267 7502 : if (OksClass::create_notify_fn)
2268 0 : set.push_back(c);
2269 : }
2270 7502 : }
2271 :
2272 182 : fp->p_size = xmls->get_position();
2273 :
2274 182 : registrate_all_classes(true);
2275 :
2276 182 : if (OksClass::create_notify_fn)
2277 0 : for (std::list<OksClass*>::iterator i2 = set.begin(); i2 != set.end(); ++i2)
2278 0 : (*OksClass::create_notify_fn)(*i2);
2279 :
2280 182 : if (OksClass::change_notify_fn) {
2281 0 : std::set<OksClass*> set2;
2282 :
2283 0 : for (std::list<OksClass*>::iterator i2 = set.begin(); i2 != set.end(); ++i2) {
2284 0 : OksClass* c = *i2;
2285 0 : if (c->p_all_sub_classes && !c->p_all_sub_classes->empty()) {
2286 0 : for (OksClass::FList::iterator i3 = c->p_all_sub_classes->begin(); i3 != c->p_all_sub_classes->end(); ++i3) {
2287 0 : if (set2.find(*i3) == set2.end()) {
2288 0 : (*OksClass::change_notify_fn)(*i3, OksClass::ChangeSuperClassesList, (const void*)(&(c->p_name)));
2289 0 : set2.insert(*i3);
2290 : }
2291 : }
2292 : }
2293 : }
2294 0 : }
2295 :
2296 182 : fp->update_status_of_file();
2297 182 : } catch (exception& e) {
2298 0 : throw FailedLoadFile("schema file", fp->get_full_file_name(), e);
2299 0 : } catch (std::exception& e) {
2300 0 : throw FailedLoadFile("schema file", fp->get_full_file_name(), e.what());
2301 0 : } catch (...) {
2302 0 : throw FailedLoadFile("schema file", fp->get_full_file_name(), "caught unknown exception");
2303 0 : }
2304 182 : }
2305 :
2306 : OksFile*
2307 0 : OksKernel::new_schema(const std::string& s)
2308 : {
2309 0 : const char _fname[] = "new_schema";
2310 0 : std::string fname = make_fname(_fname, sizeof(_fname) - 1, s, nullptr, nullptr);
2311 0 : OSK_VERBOSE_REPORT("ENTER " << fname)
2312 :
2313 0 : if (!s.length()) {
2314 0 : throw CanNotOpenFile("new_schema", s, "file name is empty");
2315 : }
2316 :
2317 0 : std::string file_name(s);
2318 :
2319 0 : try {
2320 :
2321 0 : Oks::substitute_variables(file_name);
2322 :
2323 0 : test_file_existence(file_name, p_silence, fname, "schema");
2324 :
2325 0 : file_name = get_file_path(s, nullptr, false);
2326 :
2327 0 : std::unique_lock lock(p_kernel_mutex);
2328 :
2329 0 : if (find_schema_file(file_name) != 0) {
2330 0 : throw std::runtime_error("the file is already loaded");
2331 : }
2332 :
2333 0 : OksFile* file_h = new OksFile(file_name, "", "", "schema", this);
2334 :
2335 0 : file_h->p_short_name = s;
2336 :
2337 0 : k_set_active_schema(file_h);
2338 :
2339 0 : add_schema_file(p_active_schema);
2340 :
2341 0 : registrate_all_classes(true);
2342 :
2343 0 : } catch (std::exception& e) {
2344 0 : throw CanNotCreateFile("new_schema", "schema file", file_name, e.what());
2345 0 : } catch (...) {
2346 0 : throw CanNotCreateFile("new_schema", "schema file", file_name, "caught unknown exception");
2347 0 : }
2348 :
2349 0 : OSK_VERBOSE_REPORT("LEAVE " << fname)
2350 :
2351 0 : return p_active_schema;
2352 0 : }
2353 :
2354 : // user-allowed method
2355 :
2356 : void
2357 0 : OksKernel::save_schema(OksFile* pf, bool force, OksFile* fh)
2358 : {
2359 0 : std::shared_lock lock(p_kernel_mutex);
2360 0 : k_save_schema(pf, force, fh);
2361 0 : }
2362 :
2363 : void
2364 0 : OksKernel::save_schema(OksFile* file_h, bool force, const OksClass::Map& classes)
2365 : {
2366 0 : std::shared_lock lock(p_kernel_mutex);
2367 0 : k_save_schema(file_h, force, 0, &classes);
2368 0 : }
2369 :
2370 : void
2371 0 : OksKernel::backup_schema(OksFile* pf, const char* suffix)
2372 : {
2373 0 : std::shared_lock lock(p_kernel_mutex);
2374 :
2375 0 : OksFile f(pf->get_full_file_name() + suffix, pf->get_logical_name(), pf->get_type(), pf->get_oks_format(), this);
2376 :
2377 0 : f.p_created_by = pf->p_created_by;
2378 0 : f.p_creation_time = pf->p_creation_time;
2379 0 : f.p_created_on = pf->p_created_on;
2380 0 : f.p_list_of_include_files = pf->p_list_of_include_files;
2381 :
2382 0 : if (!p_silence) {
2383 0 : std::cout << "Making backup of schema file \"" << pf->p_full_name << "\"...\n";
2384 : }
2385 :
2386 0 : bool silence = p_silence;
2387 :
2388 0 : p_silence = true;
2389 :
2390 0 : try {
2391 0 : k_save_schema(&f, true, pf);
2392 0 : } catch (exception& ex) {
2393 0 : p_silence = silence;
2394 0 : throw CanNotBackupFile(pf->p_full_name, ex);
2395 0 : }
2396 :
2397 0 : p_silence = silence;
2398 0 : }
2399 :
2400 : // kernel method
2401 :
2402 : void
2403 0 : OksKernel::k_save_schema(OksFile* pf, bool force, OksFile* fh, const OksClass::Map* classes)
2404 : {
2405 0 : const char _fname[] = "k_save_schema";
2406 0 : std::string fname = make_fname(_fname, sizeof(_fname) - 1, pf->p_full_name, nullptr, nullptr);
2407 :
2408 0 : OSK_PROFILING(OksProfiler::KernelSaveSchema, this)
2409 0 : OSK_VERBOSE_REPORT("ENTER " << fname)
2410 :
2411 0 : std::string tmp_file_name;
2412 :
2413 0 : if (!fh)
2414 0 : fh = pf;
2415 :
2416 0 : try {
2417 :
2418 : // lock the file, if it is not locked already
2419 :
2420 0 : if (pf->is_locked() == false) {
2421 0 : pf->lock();
2422 : }
2423 :
2424 : // calculate number of classes, going into the schema file
2425 :
2426 0 : size_t numberOfClasses = 0;
2427 :
2428 0 : if (classes) {
2429 0 : numberOfClasses = classes->size();
2430 0 : } else if (!p_classes.empty()) {
2431 0 : for (OksClass::Map::iterator i = p_classes.begin(); i != p_classes.end(); ++i) {
2432 0 : if (i->second->p_file == fh)
2433 0 : numberOfClasses++;
2434 : }
2435 : }
2436 :
2437 : // create temporal file to store the schema
2438 :
2439 0 : tmp_file_name = get_tmp_file(pf->p_full_name);
2440 :
2441 0 : {
2442 0 : std::ofstream f(tmp_file_name.c_str());
2443 0 : f.exceptions(std::ostream::failbit | std::ostream::badbit);
2444 :
2445 0 : if (!f) {
2446 0 : std::ostringstream text;
2447 0 : text << "cannot create temporal file \'" << tmp_file_name << "\' to save schema";
2448 0 : throw std::runtime_error(text.str().c_str());
2449 0 : } else {
2450 0 : if (!p_silence)
2451 0 : std::cout << "Saving " << numberOfClasses << " classes to schema file \"" << pf->p_full_name << "\"...\n";
2452 : }
2453 :
2454 0 : OksXmlOutputStream xmls(f);
2455 :
2456 : // write oks xml file header
2457 :
2458 0 : pf->p_number_of_items = numberOfClasses;
2459 0 : pf->p_oks_format = "schema";
2460 :
2461 0 : pf->write(xmls);
2462 :
2463 : // write oks classes
2464 :
2465 0 : std::ostringstream errors;
2466 0 : bool found_errors = false;
2467 :
2468 0 : if (classes) {
2469 0 : for (auto& i : *classes) {
2470 0 : found_errors |= i.second->check_relationships(errors, false);
2471 0 : i.second->save(xmls);
2472 : }
2473 : } else {
2474 0 : for (auto& i : p_classes) {
2475 0 : if (i.second->p_file == fh) {
2476 0 : found_errors |= i.second->check_relationships(errors, false);
2477 0 : i.second->save(xmls);
2478 : }
2479 : }
2480 : }
2481 :
2482 0 : if (found_errors) {
2483 0 : kernel::BindError ex(ERS_HERE, errors.str());
2484 :
2485 0 : if (force == false) {
2486 0 : throw std::runtime_error(ex.what());
2487 0 : } else if (p_silence == false) {
2488 0 : ers::warning(ex);
2489 : }
2490 0 : }
2491 :
2492 : // close oks xml file
2493 :
2494 0 : xmls.put_last_tag("oks-schema", sizeof("oks-schema") - 1);
2495 :
2496 : // flush the buffers
2497 :
2498 0 : f.close();
2499 0 : }
2500 :
2501 0 : if (rename(tmp_file_name.c_str(), pf->p_full_name.c_str())) {
2502 0 : std::ostringstream text;
2503 0 : text << "cannot rename \'" << tmp_file_name << "\' to \'" << pf->p_full_name << '\'';
2504 0 : throw std::runtime_error(text.str().c_str());
2505 0 : }
2506 :
2507 0 : tmp_file_name.erase(0);
2508 :
2509 0 : if (pf != p_active_schema) {
2510 0 : try {
2511 0 : pf->unlock();
2512 0 : } catch (exception& ex) {
2513 0 : throw std::runtime_error(ex.what());
2514 0 : }
2515 : }
2516 0 : pf->p_is_updated = false;
2517 0 : pf->update_status_of_file();
2518 :
2519 0 : } catch (exception& ex) {
2520 0 : if (pf != p_active_schema) {
2521 0 : try {
2522 0 : pf->unlock();
2523 0 : } catch (...) {
2524 0 : }
2525 : }
2526 0 : if (!tmp_file_name.empty()) {
2527 0 : unlink(tmp_file_name.c_str());
2528 : }
2529 0 : throw CanNotWriteToFile("k_save_schema", "schema file", pf->p_full_name, ex);
2530 0 : } catch (std::exception& ex) {
2531 0 : if (pf != p_active_schema) {
2532 0 : try {
2533 0 : pf->unlock();
2534 0 : } catch (...) {
2535 0 : }
2536 : }
2537 0 : if (!tmp_file_name.empty()) {
2538 0 : unlink(tmp_file_name.c_str());
2539 : }
2540 0 : throw CanNotWriteToFile("k_save_schema", "schema file", pf->p_full_name, ex.what());
2541 0 : }
2542 :
2543 0 : OSK_VERBOSE_REPORT("LEAVE " << fname)
2544 0 : }
2545 :
2546 : // note: the file name is used as a key in the map
2547 : // to rename a file it is necessary to remove file from map, change name and insert it back
2548 :
2549 : void
2550 0 : OksKernel::k_rename_schema(OksFile* pf, const std::string& short_name, const std::string& long_name)
2551 : {
2552 0 : remove_schema_file(pf);
2553 0 : pf->rename(short_name, long_name);
2554 0 : add_schema_file(pf);
2555 0 : }
2556 :
2557 : void
2558 0 : OksKernel::save_as_schema(const std::string& new_name, OksFile* pf)
2559 : {
2560 0 : const char _fname[] = "save_as_schema";
2561 0 : std::string fname = make_fname(_fname, sizeof(_fname) - 1, new_name, nullptr, &pf);
2562 0 : OSK_VERBOSE_REPORT("ENTER " << fname)
2563 :
2564 0 : try {
2565 :
2566 0 : if (!new_name.length()) {
2567 0 : throw std::runtime_error("the new filename is empty");
2568 : }
2569 :
2570 0 : std::string old_short_name = pf->p_short_name;
2571 0 : std::string old_full_name = pf->p_full_name;
2572 :
2573 0 : std::unique_lock lock(p_kernel_mutex);
2574 :
2575 0 : k_rename_schema(pf, new_name, new_name);
2576 :
2577 0 : try {
2578 0 : k_save_schema(pf);
2579 0 : } catch (...) {
2580 0 : k_rename_schema(pf, old_short_name, old_full_name);
2581 0 : throw;
2582 0 : }
2583 :
2584 0 : } catch (exception& ex) {
2585 0 : throw CanNotWriteToFile("k_save_as_schema", "schema file", pf->p_full_name, ex);
2586 0 : } catch (std::exception& ex) {
2587 0 : throw CanNotWriteToFile("k_save_as_schema", "schema file", pf->p_full_name, ex.what());
2588 0 : }
2589 :
2590 0 : OSK_VERBOSE_REPORT("LEAVE " << fname)
2591 0 : }
2592 :
2593 : void
2594 0 : OksKernel::save_all_schema()
2595 : {
2596 0 : OSK_VERBOSE_REPORT("ENTER OksKernel::save_all_schema()")
2597 :
2598 0 : {
2599 0 : std::shared_lock lock(p_kernel_mutex);
2600 :
2601 0 : for (OksFile::Map::iterator i = p_schema_files.begin(); i != p_schema_files.end(); ++i) {
2602 0 : if (check_read_only(i->second) == false) {
2603 0 : k_save_schema(i->second);
2604 : } else {
2605 0 : TLOG_DEBUG(2) << "skip read-only schema file \'" << *(i->first) << '\'';
2606 : }
2607 : }
2608 0 : }
2609 :
2610 0 : OSK_VERBOSE_REPORT("LEAVE OksKernel::save_all_schema()")
2611 0 : }
2612 :
2613 : // user-allowed method
2614 :
2615 : void
2616 0 : OksKernel::close_schema(OksFile* pf)
2617 : {
2618 0 : std::unique_lock lock(p_kernel_mutex);
2619 0 : k_close_schema(pf);
2620 0 : }
2621 :
2622 : // kernel method
2623 :
2624 : void
2625 182 : OksKernel::k_close_schema(OksFile* pf)
2626 : {
2627 182 : OSK_PROFILING(OksProfiler::KernelCloseSchema, this)
2628 :
2629 182 : if (!pf) {
2630 0 : TLOG_DEBUG(1) << "enter for file (null)";
2631 0 : return;
2632 : } else {
2633 182 : TLOG_DEBUG(2) << "enter for file " << pf->p_full_name;
2634 : }
2635 :
2636 182 : if (p_active_schema == pf)
2637 0 : p_active_schema = 0;
2638 :
2639 182 : try {
2640 182 : pf->unlock();
2641 0 : } catch (exception& ex) {
2642 0 : Oks::error_msg("OksKernel::k_close_schema()") << ex.what() << std::endl;
2643 0 : }
2644 :
2645 182 : if (!p_silence)
2646 0 : std::cout << (pf->p_included_by ? " * c" : "C") << "lose OKS schema \"" << pf->p_full_name << "\"..." << std::endl;
2647 :
2648 182 : if (!p_classes.empty()) {
2649 :
2650 : //
2651 : // The fastest way to delete classes is delete
2652 : // them in order of superclasses descreasing
2653 : // because this reduces amount of work to restructure
2654 : // child classes when their parent was deleted and
2655 : // there will be no dangling classes
2656 : //
2657 :
2658 182 : std::set<OksClass*> sorted;
2659 :
2660 12271 : for (OksClass::Map::iterator i = p_classes.begin(); i != p_classes.end(); ++i) {
2661 12089 : OksClass* c = i->second;
2662 12089 : if (pf == c->p_file)
2663 7502 : sorted.insert(c);
2664 12089 : c->p_to_be_deleted = true;
2665 : }
2666 :
2667 7684 : for (std::set<OksClass*>::iterator j = sorted.begin(); j != sorted.end(); ++j) {
2668 7502 : delete *j;
2669 : }
2670 182 : }
2671 :
2672 182 : remove_schema_file(pf);
2673 182 : delete pf;
2674 :
2675 182 : TLOG_DEBUG(4) << "exit for file " << (void*)pf;
2676 182 : }
2677 :
2678 : void
2679 190 : OksKernel::close_all_schema()
2680 : {
2681 190 : TLOG_DEBUG(4) << "enter";
2682 :
2683 190 : {
2684 190 : std::unique_lock lock(p_kernel_mutex);
2685 :
2686 7692 : for (OksClass::Map::iterator i = p_classes.begin(); i != p_classes.end(); ++i) {
2687 7502 : i->second->p_to_be_deleted = true;
2688 : }
2689 :
2690 372 : while (!p_schema_files.empty()) {
2691 182 : k_close_schema(p_schema_files.begin()->second);
2692 : }
2693 190 : }
2694 :
2695 190 : TLOG_DEBUG(4) << "exit";
2696 190 : }
2697 :
2698 : void
2699 0 : OksKernel::set_active_schema(OksFile* f)
2700 : {
2701 0 : std::unique_lock lock(p_kernel_mutex);
2702 0 : k_set_active_schema(f);
2703 0 : }
2704 :
2705 : void
2706 0 : OksKernel::k_set_active_schema(OksFile* f)
2707 : {
2708 0 : TLOG_DEBUG(4) << "enter for file " << (void*)f;
2709 :
2710 : // check if active schema is different from given file
2711 :
2712 0 : if (p_active_schema == f)
2713 : return;
2714 :
2715 : // unlock current active schema, if it was saved
2716 :
2717 0 : if (p_active_schema && p_active_schema->is_updated() == false) {
2718 0 : try {
2719 0 : p_active_schema->unlock();
2720 0 : } catch (exception& ex) {
2721 0 : throw CanNotSetActiveFile("schema", f->get_full_file_name(), ex);
2722 0 : }
2723 : }
2724 :
2725 : // exit, if file is (null)
2726 :
2727 0 : if (!f) {
2728 0 : p_active_schema = 0;
2729 0 : return;
2730 : }
2731 :
2732 0 : try {
2733 0 : f->lock();
2734 0 : p_active_schema = f;
2735 0 : } catch (exception& ex) {
2736 0 : throw CanNotSetActiveFile("schema", f->get_full_file_name(), ex);
2737 0 : }
2738 :
2739 0 : TLOG_DEBUG(4) << "exit for file " << (void*)f;
2740 : }
2741 :
2742 : std::list<OksClass*>*
2743 0 : OksKernel::create_list_of_schema_classes(OksFile* pf) const
2744 : {
2745 0 : std::string fname("OksKernel::create_list_of_schema_classes(");
2746 0 : fname.append(pf->get_full_file_name());
2747 0 : fname.push_back('\'');
2748 :
2749 0 : OSK_PROFILING(OksProfiler::KernelCreateSchemaClassList, this)
2750 0 : OSK_VERBOSE_REPORT("ENTER " << fname)
2751 :
2752 0 : std::list<OksClass*>* clist = nullptr;
2753 :
2754 0 : if (!p_classes.empty()) {
2755 0 : for (OksClass::Map::const_iterator i = p_classes.begin(); i != p_classes.end(); ++i) {
2756 0 : if (pf == i->second->p_file) {
2757 0 : if (!clist && !(clist = new std::list<OksClass*>())) {
2758 : return 0;
2759 : }
2760 :
2761 0 : clist->push_back(i->second);
2762 : }
2763 : }
2764 : }
2765 :
2766 0 : OSK_VERBOSE_REPORT("LEAVE " << fname)
2767 :
2768 0 : return clist;
2769 0 : }
2770 :
2771 : /******************************************************************************/
2772 :
2773 : static void
2774 6 : create_updated_lists(const OksFile::Map& files,
2775 : std::list<OksFile*>** ufs,
2776 : std::list<OksFile*>** rfs,
2777 : OksFile::FileStatus wu,
2778 : OksFile::FileStatus wr)
2779 : {
2780 6 : *ufs = 0; // updated files
2781 6 : *rfs = 0; // removed files
2782 :
2783 12 : for (OksFile::Map::const_iterator i = files.begin(); i != files.end(); ++i) {
2784 6 : OksFile::FileStatus fs = i->second->get_status_of_file();
2785 :
2786 6 : if (fs == wu /* i.e. Which Updated */) {
2787 0 : if (*ufs == 0) {
2788 0 : *ufs = new std::list<OksFile*>();
2789 : }
2790 0 : (*ufs)->push_back(i->second);
2791 6 : } else if (fs == wr /* i.e. Which Removed */) {
2792 0 : if (*rfs == 0) {
2793 0 : *rfs = new std::list<OksFile*>();
2794 : }
2795 0 : (*rfs)->push_back(i->second);
2796 : }
2797 : }
2798 6 : }
2799 :
2800 : void
2801 3 : OksKernel::create_lists_of_updated_schema_files(std::list<OksFile*>** ufs, std::list<OksFile*>** rfs) const
2802 : {
2803 3 : std::shared_lock lock(p_kernel_mutex);
2804 :
2805 3 : create_updated_lists(p_schema_files, ufs, rfs, OksFile::FileModified, OksFile::FileRemoved);
2806 3 : }
2807 :
2808 : void
2809 3 : OksKernel::create_lists_of_updated_data_files(std::list<OksFile*>** ufs, std::list<OksFile*>** rfs) const
2810 : {
2811 3 : std::shared_lock lock(p_kernel_mutex);
2812 :
2813 3 : create_updated_lists(p_data_files, ufs, rfs, OksFile::FileModified, OksFile::FileRemoved);
2814 3 : }
2815 :
2816 : void
2817 3 : OksKernel::get_modified_files(std::set<OksFile*>& mfs, std::set<OksFile*>& rfs, const std::string& version)
2818 : {
2819 3 : std::list<OksFile*>* files[] = { nullptr, nullptr, nullptr, nullptr };
2820 :
2821 3 : std::set<OksFile*>* sets[] = { &mfs, &rfs };
2822 :
2823 3 : create_lists_of_updated_data_files(&files[0], &files[1]);
2824 3 : create_lists_of_updated_schema_files(&files[2], &files[3]);
2825 :
2826 15 : for (int i = 0; i < 4; ++i)
2827 12 : if (std::list<OksFile*>* f = files[i]) {
2828 0 : while (!f->empty()) {
2829 0 : sets[i % 2]->insert(f->front());
2830 0 : f->pop_front();
2831 : }
2832 0 : delete f;
2833 : }
2834 :
2835 3 : if (!get_user_repository_root().empty() && !version.empty())
2836 0 : for (const auto& file_name : get_repository_versions_diff(get_repository_version(), version)) {
2837 0 : if (OksFile* f = find_data_file(file_name))
2838 0 : sets[0]->insert(f);
2839 0 : else if (OksFile* f = find_schema_file(file_name))
2840 0 : sets[0]->insert(f);
2841 0 : }
2842 3 : }
2843 :
2844 : /******************************************************************************/
2845 :
2846 : void
2847 0 : LoadErrors::add_parents(std::string& text, const OksFile* file, std::set<const OksFile*>& parents)
2848 : {
2849 0 : if (file) {
2850 0 : if (parents.insert(file).second == false) {
2851 0 : text += "(ignoring circular dependency between included files...)\n";
2852 : } else {
2853 0 : add_parents(text, file->get_parent(), parents);
2854 0 : text += "file \'";
2855 0 : text += file->get_full_file_name();
2856 0 : text += "\' includes:\n";
2857 : }
2858 : }
2859 0 : }
2860 :
2861 : void
2862 0 : LoadErrors::add_error(const OksFile& file, std::exception& ex)
2863 : {
2864 0 : std::string text;
2865 0 : std::set<const OksFile*> parents;
2866 0 : add_parents(text, file.get_parent(), parents);
2867 :
2868 0 : bool has_no_parents(text.empty());
2869 :
2870 0 : text += "file \'";
2871 0 : text += file.get_full_file_name();
2872 0 : text += (has_no_parents ? "\' has problem:\n" : "\' that has problem:\n");
2873 0 : text += ex.what();
2874 :
2875 0 : std::lock_guard lock(p_mutex);
2876 0 : m_errors.push_back(text);
2877 0 : }
2878 :
2879 : std::string
2880 0 : LoadErrors::get_text()
2881 : {
2882 0 : std::lock_guard lock(p_mutex);
2883 :
2884 0 : {
2885 0 : std::ostringstream s;
2886 :
2887 0 : if (m_errors.size() == 1) {
2888 0 : s << "Found 1 error parsing OKS data:\n" << *m_errors.begin();
2889 : } else {
2890 0 : s << "Found " << m_errors.size() << " errors parsing OKS data:";
2891 0 : int j = 1;
2892 0 : for (std::list<std::string>::const_iterator i = m_errors.begin(); i != m_errors.end(); ++i) {
2893 0 : s << "\nERROR [" << j++ << "] ***: " << *i;
2894 : }
2895 : }
2896 :
2897 0 : m_error_string = s.str();
2898 0 : }
2899 :
2900 0 : return m_error_string;
2901 0 : }
2902 :
2903 : struct OksLoadObjectsJob : public OksJob
2904 : {
2905 : public:
2906 133 : OksLoadObjectsJob(OksKernel* kernel, OksFile* fp, std::shared_ptr<OksXmlInputStream> xmls, char format)
2907 133 : : m_kernel(kernel)
2908 133 : , m_fp(fp)
2909 133 : , m_xmls(xmls)
2910 133 : , m_format(format)
2911 : {
2912 133 : ;
2913 133 : }
2914 :
2915 133 : void run()
2916 : {
2917 133 : try {
2918 133 : OksAliasTable alias_table;
2919 133 : ReadFileParams read_params(m_fp, *m_xmls, ((m_format == 'X') ? 0 : &alias_table), m_kernel, m_format, 0);
2920 :
2921 133 : m_fp->p_number_of_items = 0;
2922 :
2923 133 : try {
2924 5577 : while (OksObject::read(read_params)) {
2925 5444 : m_fp->p_number_of_items++;
2926 : }
2927 0 : } catch (FailedCreateObject& ex) {
2928 0 : m_kernel->p_load_errors.add_error(*m_fp, ex);
2929 0 : return;
2930 0 : }
2931 :
2932 133 : m_fp->p_size = m_xmls->get_position();
2933 133 : } catch (std::exception& ex) {
2934 0 : m_kernel->p_load_errors.add_error(*m_fp, ex);
2935 0 : }
2936 : }
2937 :
2938 : private:
2939 : OksKernel* m_kernel;
2940 : OksFile* m_fp;
2941 : std::shared_ptr<OksXmlInputStream> m_xmls;
2942 : char m_format;
2943 :
2944 : // protect usage of copy constructor and assignment operator
2945 :
2946 : private:
2947 : OksLoadObjectsJob(const OksLoadObjectsJob&);
2948 : OksLoadObjectsJob& operator=(const OksLoadObjectsJob&);
2949 : };
2950 :
2951 : /******************************************************************************/
2952 :
2953 : static bool
2954 0 : _find_file(const OksFile::Map& files, const OksFile* f)
2955 : {
2956 0 : for (OksFile::Map::const_iterator i = files.begin(); i != files.end(); ++i) {
2957 0 : if (i->second == f)
2958 0 : return true;
2959 : }
2960 0 : return false;
2961 : }
2962 :
2963 : void
2964 0 : ReloadObjects::put(OksObject* o)
2965 : {
2966 0 : map_str_t<OksObject*>*& c = data[o->uid.class_id];
2967 0 : if (!c)
2968 0 : c = new map_str_t<OksObject*>();
2969 0 : (*c)[o->GetId()] = o;
2970 0 : }
2971 :
2972 : OksObject*
2973 0 : ReloadObjects::pop(const OksClass* c, const std::string& id)
2974 : {
2975 0 : std::map<const OksClass*, map_str_t<OksObject*>*>::iterator i = data.find(c);
2976 0 : if (i != data.end()) {
2977 0 : map_str_t<OksObject*>::iterator j = i->second->find(id);
2978 0 : if (j != i->second->end()) {
2979 0 : OksObject* o = j->second;
2980 0 : i->second->erase(j);
2981 0 : if (i->second->empty()) {
2982 0 : delete i->second;
2983 0 : data.erase(i);
2984 : }
2985 0 : return o;
2986 : }
2987 : }
2988 :
2989 : return 0;
2990 : }
2991 :
2992 0 : ReloadObjects::~ReloadObjects()
2993 : {
2994 0 : for (std::map<const OksClass*, map_str_t<OksObject*>*>::iterator i = data.begin(); i != data.end(); ++i) {
2995 0 : delete i->second;
2996 : }
2997 0 : }
2998 :
2999 : void
3000 0 : OksKernel::reload_data(std::set<OksFile*>& files_h, bool allow_schema_extension)
3001 : {
3002 0 : std::map<OksFile*, std::vector<std::string>> included;
3003 :
3004 0 : std::set<OksFile*>::const_iterator i;
3005 :
3006 0 : bool check_includes(false);
3007 0 : bool found_schema_files(false);
3008 0 : std::string file_names;
3009 :
3010 0 : std::unique_lock lock(p_kernel_mutex);
3011 :
3012 0 : for (std::set<OksFile*>::const_iterator fi = files_h.begin(); fi != files_h.end();) {
3013 0 : if (_find_file(p_schema_files, *fi)) {
3014 : found_schema_files = true;
3015 0 : } else if (!_find_file(p_data_files, *fi)) {
3016 0 : Oks::error_msg("OksKernel::reload_data")
3017 0 : << "file " << (void*)(*fi) << " is not OKS data or schema file, skip..." << std::endl;
3018 0 : files_h.erase(fi++);
3019 0 : continue;
3020 : }
3021 :
3022 0 : if (fi != files_h.begin())
3023 0 : file_names.append(", ");
3024 0 : file_names.push_back('\"');
3025 0 : file_names.append((*fi)->get_full_file_name());
3026 0 : file_names.push_back('\"');
3027 0 : ++fi;
3028 : }
3029 :
3030 0 : try {
3031 :
3032 : // throw exception on schema_files
3033 :
3034 0 : if (found_schema_files) {
3035 0 : throw std::runtime_error("Reload of modified schema files is not supported");
3036 : }
3037 :
3038 : // exit, if there are no files to reload (e.g. dangling refs.)
3039 :
3040 0 : if (files_h.empty())
3041 0 : return;
3042 :
3043 : // unlock any locked file
3044 : // build container of objects which can be updated (and removed, if not reloaded)
3045 :
3046 0 : ReloadObjects reload_objects;
3047 :
3048 0 : for (i = files_h.begin(); i != files_h.end(); ++i) {
3049 0 : if ((*i)->is_locked()) {
3050 0 : if (p_active_data == (*i)) {
3051 0 : p_active_data = 0;
3052 : }
3053 0 : (*i)->unlock();
3054 : }
3055 :
3056 0 : for (OksObject::Set::const_iterator oi = p_objects.begin(); oi != p_objects.end(); ++oi) {
3057 0 : if ((*oi)->file == *i)
3058 0 : reload_objects.put(*oi);
3059 : }
3060 : }
3061 :
3062 : // preload includes of modified files
3063 :
3064 0 : {
3065 0 : std::set<OksFile*> new_files;
3066 :
3067 0 : for (i = files_h.begin(); i != files_h.end(); ++i) {
3068 0 : try {
3069 0 : check_includes |= k_preload_includes(*i, new_files, allow_schema_extension);
3070 0 : } catch (...) {
3071 0 : restore_preload_file_info();
3072 0 : throw;
3073 0 : }
3074 : }
3075 :
3076 0 : clear_preload_file_info();
3077 :
3078 0 : if (!new_files.empty()) {
3079 0 : TLOG_DEBUG(2) << new_files.size() << " new files will be loaded";
3080 : }
3081 :
3082 0 : for (i = new_files.begin(); i != new_files.end(); ++i) {
3083 0 : files_h.insert(*i);
3084 : }
3085 0 : }
3086 :
3087 : // build list of files to be closed:
3088 : // close files which are no more referenced by others
3089 : // this may happen if there are changes in the list of includes
3090 :
3091 0 : std::set<OksFile*> files_to_be_closed;
3092 :
3093 0 : if (check_includes) {
3094 : size_t num_of_closing_files = 0;
3095 :
3096 0 : while (true) {
3097 :
3098 : // build list of good (i.e. "included") files
3099 : // rebuild the list, if at least one file was excluded
3100 :
3101 0 : std::map<std::string, OksFile*> good_files;
3102 :
3103 0 : for (OksFile::Map::iterator j = p_data_files.begin(); j != p_data_files.end(); ++j) {
3104 0 : if (files_to_be_closed.find(j->second) == files_to_be_closed.end()) {
3105 0 : for (std::list<std::string>::iterator l = j->second->p_list_of_include_files.begin();
3106 0 : l != j->second->p_list_of_include_files.end();
3107 0 : ++l) {
3108 0 : try {
3109 0 : good_files[get_file_path(*l, j->second)] = j->second;
3110 0 : } catch (...) {
3111 0 : }
3112 : }
3113 : }
3114 : }
3115 :
3116 0 : for (OksFile::Map::iterator i = p_data_files.begin(); i != p_data_files.end();) {
3117 0 : if (i->second->p_included_by && files_to_be_closed.find(i->second) == files_to_be_closed.end()) {
3118 0 : TLOG_DEBUG(3) << "check the file \'" << i->second->get_full_file_name() << "\' is included";
3119 0 : const std::string& s = i->second->get_full_file_name();
3120 0 : OksFile* f2 = i->second;
3121 0 : ++i;
3122 :
3123 0 : f2->p_included_by = 0;
3124 :
3125 : // search for a file which could include given one
3126 :
3127 0 : std::map<std::string, OksFile*>::const_iterator j = good_files.find(s);
3128 :
3129 0 : if (j != good_files.end()) {
3130 0 : f2->p_included_by = j->second;
3131 : } else {
3132 0 : files_to_be_closed.insert(f2);
3133 :
3134 0 : for (OksObject::Set::const_iterator oi = p_objects.begin(); oi != p_objects.end(); ++oi) {
3135 0 : if ((*oi)->file == f2) {
3136 0 : reload_objects.put(*oi);
3137 : }
3138 : }
3139 :
3140 0 : if (files_h.erase(f2)) {
3141 0 : TLOG_DEBUG(2) << "skip reload of updated file \'" << f2->get_full_file_name()
3142 0 : << " since it will be closed";
3143 : }
3144 :
3145 0 : TLOG_DEBUG(3) << "file \'" << f2->get_full_file_name() << " will be closed";
3146 : }
3147 : } else {
3148 0 : ++i;
3149 : }
3150 : }
3151 :
3152 0 : if (num_of_closing_files == files_to_be_closed.size()) {
3153 : break;
3154 : } else {
3155 0 : num_of_closing_files = files_to_be_closed.size();
3156 : }
3157 0 : }
3158 : } else {
3159 0 : TLOG_DEBUG(2) << "no changes in the list of includes";
3160 : }
3161 :
3162 : // remove exclusive RCRs (will be restored when read, if object was not changed)
3163 :
3164 0 : {
3165 0 : for (std::map<const OksClass*, map_str_t<OksObject*>*>::const_iterator cx = reload_objects.data.begin();
3166 0 : cx != reload_objects.data.end();
3167 0 : ++cx) {
3168 0 : const OksClass* c(cx->first);
3169 0 : if (c->p_all_relationships && !c->p_all_relationships->empty()) {
3170 0 : const unsigned int atts_num(c->number_of_all_attributes());
3171 0 : for (map_str_t<OksObject*>::const_iterator j = cx->second->begin(); j != cx->second->end(); ++j) {
3172 0 : OksObject* obj = j->second;
3173 0 : OksData* d(obj->data + atts_num);
3174 :
3175 0 : for (std::list<OksRelationship*>::iterator i = c->p_all_relationships->begin();
3176 0 : i != c->p_all_relationships->end();
3177 0 : ++i, ++d) {
3178 0 : OksRelationship* r = *i;
3179 0 : if (r->get_is_composite() && r->get_is_exclusive()) {
3180 0 : if (d->type == OksData::object_type && d->data.OBJECT) {
3181 0 : if (!is_dangling(d->data.OBJECT)) {
3182 0 : d->data.OBJECT->remove_RCR(obj, r);
3183 : }
3184 0 : } else if (d->type == OksData::list_type && d->data.LIST) {
3185 0 : for (OksData::List::iterator i2 = d->data.LIST->begin(); i2 != d->data.LIST->end(); ++i2) {
3186 0 : OksData* d2(*i2);
3187 0 : if (d2->type == OksData::object_type && d2->data.OBJECT) {
3188 0 : if (!is_dangling(d2->data.OBJECT)) {
3189 0 : d2->data.OBJECT->remove_RCR(obj, r);
3190 : }
3191 : }
3192 : }
3193 : }
3194 : }
3195 : }
3196 : }
3197 : }
3198 : }
3199 : }
3200 :
3201 : // need to allow "duplicated_objects_via_inheritance_mode", since the objects with the same ID can be created and
3202 : // removed
3203 :
3204 0 : bool duplicated_objs_mode = get_test_duplicated_objects_via_inheritance_mode();
3205 0 : set_test_duplicated_objects_via_inheritance_mode(false);
3206 :
3207 : // read objects
3208 :
3209 0 : for (i = files_h.begin(); i != files_h.end(); ++i) {
3210 0 : std::shared_ptr<std::ifstream> f(new std::ifstream((*i)->get_full_file_name().c_str()));
3211 :
3212 0 : std::shared_ptr<OksXmlInputStream> xmls(new OksXmlInputStream(f));
3213 :
3214 0 : OksFile fp(xmls, (*i)->get_short_file_name(), (*i)->get_full_file_name(), this);
3215 0 : char format = (fp.p_oks_format == "data" ? 'n' : ((fp.p_oks_format == "extended") ? 'X' : 'c'));
3216 :
3217 0 : if (!p_silence) {
3218 0 : std::lock_guard lock(p_parallel_out_mutex);
3219 0 : std::cout << " * reading data file \"" << (*i)->get_full_file_name() << "\"..." << std::endl;
3220 0 : }
3221 :
3222 0 : {
3223 0 : OksAliasTable alias_table;
3224 0 : ReadFileParams read_params(*i, *xmls, ((format == 'X') ? 0 : &alias_table), this, format, &reload_objects);
3225 :
3226 : try {
3227 0 : while (OksObject::read(read_params)) {
3228 : ;
3229 : }
3230 0 : } catch (FailedCreateObject& ex) {
3231 0 : set_test_duplicated_objects_via_inheritance_mode(duplicated_objs_mode);
3232 0 : throw ex;
3233 0 : }
3234 0 : }
3235 :
3236 0 : (*i)->update_status_of_file();
3237 0 : (*i)->p_is_updated = false;
3238 0 : }
3239 :
3240 0 : set_test_duplicated_objects_via_inheritance_mode(duplicated_objs_mode);
3241 :
3242 : // remove objects which were not re-read
3243 :
3244 0 : OksObject::FSet oset;
3245 :
3246 0 : {
3247 0 : for (std::map<const OksClass*, map_str_t<OksObject*>*>::const_iterator cx = reload_objects.data.begin();
3248 0 : cx != reload_objects.data.end();
3249 0 : ++cx) {
3250 0 : for (map_str_t<OksObject*>::const_iterator ox = cx->second->begin(); ox != cx->second->end(); ++ox) {
3251 0 : oset.insert(ox->second);
3252 : }
3253 : }
3254 :
3255 : #ifndef ERS_NO_DEBUG
3256 0 : if (ers::debug_level() >= 3) {
3257 0 : std::ostringstream text;
3258 0 : text << "there are " << oset.size() << " removed objects:\n";
3259 :
3260 0 : for (OksObject::FSet::iterator x = oset.begin(); x != oset.end(); ++x) {
3261 0 : text << " - object " << *x << std::endl;
3262 0 : TLOG_DEBUG(3) << text.str();
3263 : }
3264 0 : }
3265 : #endif
3266 :
3267 0 : {
3268 0 : OksObject::FSet refs;
3269 0 : unbind_all_rels(oset, refs);
3270 0 : if (p_change_object_notify_fn) {
3271 0 : for (OksObject::FSet::const_iterator x2 = refs.begin(); x2 != refs.end(); ++x2) {
3272 0 : TLOG_DEBUG(3) << "*** add object " << *x2 << " to the list of updated *** ";
3273 0 : (*p_change_object_notify_fn)(*x2, p_change_object_notify_param);
3274 : }
3275 : }
3276 0 : }
3277 : }
3278 :
3279 0 : for (OksObject::FSet::iterator ox = oset.begin(); ox != oset.end(); ++ox) {
3280 0 : OksObject* o = *ox;
3281 :
3282 0 : if (is_dangling(o)) {
3283 0 : TLOG_DEBUG(4) << "skip dangling object " << (void*)o;
3284 0 : continue;
3285 0 : }
3286 :
3287 0 : TLOG_DEBUG(3) << "*** remove non-reloaded object " << o << " => " << (void*)o << " ***";
3288 :
3289 0 : delete o;
3290 : }
3291 :
3292 0 : for (std::set<OksFile*>::const_iterator x = files_to_be_closed.begin(); x != files_to_be_closed.end(); ++x) {
3293 0 : k_close_data(*x, true);
3294 : }
3295 :
3296 0 : k_bind_objects();
3297 :
3298 : // check that created objects do not have duplicated IDs within inheritance hierarchy
3299 :
3300 0 : if (duplicated_objs_mode == true) {
3301 0 : for (auto& o : reload_objects.created) {
3302 0 : o->check_ids();
3303 : }
3304 : }
3305 0 : } catch (exception& e) {
3306 0 : throw FailedReloadFile(file_names, e);
3307 0 : } catch (std::exception& e) {
3308 0 : throw FailedReloadFile(file_names, e.what());
3309 0 : }
3310 0 : }
3311 :
3312 : void
3313 0 : OksKernel::clear_preload_file_info()
3314 : {
3315 0 : for (std::map<const OksFile*, OksFile*>::iterator i = p_preload_file_info.begin(); i != p_preload_file_info.end();
3316 0 : ++i) {
3317 0 : delete i->second;
3318 : }
3319 :
3320 0 : p_preload_file_info.clear();
3321 0 : p_preload_added_files.clear();
3322 0 : }
3323 :
3324 : void
3325 0 : OksKernel::restore_preload_file_info()
3326 : {
3327 0 : for (std::vector<OksFile*>::iterator j = p_preload_added_files.begin(); j != p_preload_added_files.end(); ++j) {
3328 0 : TLOG_DEBUG(1) << "remove file " << (*j)->get_full_file_name();
3329 0 : remove_data_file(*j);
3330 0 : delete *j;
3331 : }
3332 :
3333 0 : for (OksFile::Map::iterator i = p_data_files.begin(); i != p_data_files.end(); ++i) {
3334 0 : std::map<const OksFile*, OksFile*>::iterator j = p_preload_file_info.find(i->second);
3335 0 : if (j != p_preload_file_info.end()) {
3336 0 : TLOG_DEBUG(1) << "restore file " << i->second->get_full_file_name();
3337 0 : (*i->second) = (*j->second);
3338 : }
3339 : }
3340 :
3341 0 : clear_preload_file_info();
3342 0 : }
3343 :
3344 : // user-allowed method
3345 :
3346 : OksFile*
3347 0 : OksKernel::load_data(const std::string& short_file_name, bool bind)
3348 : {
3349 0 : std::unique_lock lock(p_kernel_mutex);
3350 0 : return k_load_data(short_file_name, bind, 0, 0);
3351 0 : }
3352 :
3353 : // kernel method
3354 :
3355 : OksFile*
3356 0 : OksKernel::k_load_data(const std::string& short_file_name, bool bind, const OksFile* parent_h, OksPipeline* pipeline)
3357 : {
3358 0 : const char _fname[] = "k_load_data";
3359 0 : std::string fname = make_fname(_fname, sizeof(_fname) - 1, short_file_name, &bind, &parent_h);
3360 :
3361 0 : OSK_VERBOSE_REPORT("ENTER " << fname)
3362 :
3363 0 : std::string full_file_name;
3364 :
3365 0 : try {
3366 0 : full_file_name = get_file_path(short_file_name, parent_h);
3367 0 : } catch (std::exception& e) {
3368 0 : throw CanNotOpenFile("k_load_data", short_file_name, e.what());
3369 0 : }
3370 :
3371 0 : try {
3372 :
3373 0 : OksFile* fp = find_data_file(full_file_name);
3374 :
3375 0 : if (fp != 0) {
3376 0 : if (p_verbose) {
3377 0 : std::lock_guard lock(p_parallel_out_mutex);
3378 0 : Oks::warning_msg(fname) << " The file \'" << full_file_name << "\' was already loaded";
3379 0 : }
3380 0 : return fp->check_parent(parent_h);
3381 : }
3382 :
3383 0 : {
3384 0 : std::shared_ptr<std::ifstream> f(new std::ifstream(full_file_name.c_str()));
3385 :
3386 0 : if (!f->good()) {
3387 0 : throw std::runtime_error("k_load_data(): cannot open file");
3388 : }
3389 :
3390 0 : long file_length(get_file_length(*f));
3391 :
3392 0 : if (!file_length) {
3393 0 : throw std::runtime_error("k_load_data(): file is empty");
3394 : }
3395 :
3396 0 : std::shared_ptr<OksXmlInputStream> xmls(new OksXmlInputStream(f));
3397 :
3398 0 : fp = new OksFile(xmls, short_file_name, full_file_name, this);
3399 :
3400 0 : if (fp->p_oks_format.empty()) {
3401 0 : throw std::runtime_error("k_load_data(): failed to read header of file");
3402 : }
3403 :
3404 0 : char format;
3405 :
3406 0 : if (fp->p_oks_format.size() == 4 && cmp_str4n(fp->p_oks_format.c_str(), "data"))
3407 : format = 'n';
3408 0 : else if (fp->p_oks_format.size() == 8 && cmp_str8n(fp->p_oks_format.c_str(), "extended"))
3409 : format = 'X';
3410 0 : else if (fp->p_oks_format.size() == 7 && cmp_str7n(fp->p_oks_format.c_str(), "compact"))
3411 : format = 'c';
3412 : else {
3413 0 : throw std::runtime_error("k_load_data(): file is not an oks data file");
3414 : }
3415 :
3416 0 : k_load_data(fp, format, xmls, file_length, bind, parent_h, pipeline);
3417 :
3418 0 : OSK_VERBOSE_REPORT("LEAVE " << fname)
3419 :
3420 0 : return fp;
3421 0 : }
3422 0 : } catch (FailedLoadFile&) {
3423 0 : throw;
3424 0 : } catch (exception& e) {
3425 0 : throw(FailedLoadFile("data file", full_file_name, e));
3426 0 : } catch (std::exception& e) {
3427 0 : throw(FailedLoadFile("data file", full_file_name, e.what()));
3428 0 : }
3429 0 : }
3430 :
3431 : void
3432 133 : OksKernel::k_load_data(OksFile* fp,
3433 : char format,
3434 : std::shared_ptr<OksXmlInputStream> xmls,
3435 : long file_length,
3436 : bool bind,
3437 : const OksFile* parent_h,
3438 : OksPipeline* pipeline)
3439 : {
3440 133 : OSK_PROFILING(OksProfiler::KernelLoadData, this)
3441 :
3442 133 : fp->p_included_by = parent_h;
3443 133 : check_read_only(fp);
3444 :
3445 133 : try {
3446 133 : if (!p_silence) {
3447 0 : std::lock_guard lock(p_parallel_out_mutex);
3448 0 : std::cout << (parent_h ? " * r" : "R") << "eading data file \"" << fp->get_full_file_name() << "\" in "
3449 0 : << (format == 'n' ? "normal" : (format == 'X' ? "extended" : "compact")) << " format (" << file_length
3450 0 : << " bytes)...\n";
3451 0 : if (parent_h == 0 && fp->get_full_file_name() != fp->get_short_file_name()) {
3452 0 : std::cout << "(non-fully-qualified filename was \"" << fp->get_short_file_name() << "\")\n";
3453 : }
3454 0 : }
3455 :
3456 133 : fp->update_status_of_file();
3457 133 : add_data_file(fp);
3458 :
3459 133 : {
3460 133 : std::unique_ptr<OksPipeline> pipeline_guard(pipeline ? 0 : new OksPipeline(p_threads_pool_size));
3461 :
3462 133 : OksPipeline* m_pipeline;
3463 :
3464 133 : if (pipeline) {
3465 : m_pipeline = pipeline;
3466 : } else {
3467 90 : m_pipeline = pipeline_guard.get();
3468 90 : p_load_errors.clear();
3469 : }
3470 :
3471 133 : k_load_includes(*fp, m_pipeline);
3472 :
3473 133 : if (p_threads_pool_size > 1) {
3474 133 : m_pipeline->addJob(new OksLoadObjectsJob(this, fp, xmls, format));
3475 : } else {
3476 0 : OksLoadObjectsJob job(this, fp, xmls, format);
3477 0 : job.run();
3478 0 : }
3479 133 : }
3480 :
3481 133 : if (!pipeline) {
3482 90 : if (!p_load_errors.is_empty()) {
3483 0 : throw(FailedLoadFile("data file", fp->get_full_file_name(), p_load_errors.get_text()));
3484 : }
3485 :
3486 90 : if (bind) {
3487 90 : k_bind_objects();
3488 : }
3489 : }
3490 0 : } catch (FailedLoadFile&) {
3491 0 : throw;
3492 0 : } catch (exception& e) {
3493 0 : throw(FailedLoadFile("data file", fp->get_full_file_name(), e));
3494 0 : } catch (std::exception& e) {
3495 0 : throw(FailedLoadFile("data file", fp->get_full_file_name(), e.what()));
3496 0 : }
3497 133 : }
3498 :
3499 : OksFile*
3500 5 : OksKernel::new_data(const std::string& s, const std::string& logical_name, const std::string& type)
3501 : {
3502 5 : const char _fname[] = "new_data";
3503 5 : std::string fname = make_fname(_fname, sizeof(_fname) - 1, s, nullptr, nullptr);
3504 5 : OSK_VERBOSE_REPORT("ENTER " << fname)
3505 :
3506 5 : if (!s.length()) {
3507 0 : throw CanNotOpenFile("new_data", s, "file name is empty");
3508 : }
3509 :
3510 5 : std::string file_name(s);
3511 :
3512 5 : try {
3513 :
3514 5 : Oks::substitute_variables(file_name);
3515 :
3516 5 : test_file_existence(file_name, p_silence, fname, "data");
3517 :
3518 5 : file_name = get_file_path(s, nullptr, false);
3519 :
3520 5 : std::unique_lock lock(p_kernel_mutex);
3521 :
3522 5 : if (find_data_file(file_name) != 0) {
3523 0 : throw std::runtime_error("the file is already loaded");
3524 : }
3525 :
3526 5 : OksFile* file_h = new OksFile(file_name, logical_name, type, "data", this);
3527 :
3528 5 : file_h->p_short_name = s;
3529 :
3530 5 : k_set_active_data(file_h);
3531 :
3532 5 : add_data_file(p_active_data);
3533 :
3534 5 : } catch (exception& e) {
3535 0 : throw CanNotCreateFile("new_data", "data file", file_name, e);
3536 0 : } catch (std::exception& e) {
3537 0 : throw CanNotCreateFile("new_data", "data file", file_name, e.what());
3538 0 : }
3539 :
3540 5 : OSK_VERBOSE_REPORT("LEAVE " << fname)
3541 :
3542 5 : return p_active_data;
3543 5 : }
3544 :
3545 : void
3546 138 : OksKernel::add_data_file(OksFile* f)
3547 : {
3548 138 : p_data_files[&f->p_full_name] = f;
3549 138 : }
3550 :
3551 : void
3552 182 : OksKernel::add_schema_file(OksFile* f)
3553 : {
3554 182 : p_schema_files[&f->p_full_name] = f;
3555 182 : }
3556 :
3557 : void
3558 138 : OksKernel::remove_data_file(OksFile* f)
3559 : {
3560 138 : p_data_files.erase(&f->p_full_name);
3561 138 : }
3562 :
3563 : void
3564 182 : OksKernel::remove_schema_file(OksFile* f)
3565 : {
3566 182 : p_schema_files.erase(&f->p_full_name);
3567 182 : }
3568 :
3569 : // user-allowed method
3570 :
3571 : void
3572 0 : OksKernel::save_data(OksFile* pf, bool ignoreBadObjects, OksFile* true_file_h, bool force_defaults)
3573 : {
3574 0 : std::shared_lock lock(p_kernel_mutex);
3575 0 : k_save_data(pf, ignoreBadObjects, true_file_h, nullptr, force_defaults);
3576 0 : }
3577 :
3578 : void
3579 0 : OksKernel::save_data(OksFile* file_h, const OksObject::FSet& objects)
3580 : {
3581 0 : std::shared_lock lock(p_kernel_mutex);
3582 0 : k_save_data(file_h, false, 0, &objects);
3583 0 : }
3584 :
3585 : void
3586 0 : OksKernel::backup_data(OksFile* pf, const char* suffix)
3587 : {
3588 0 : std::shared_lock lock(p_kernel_mutex);
3589 :
3590 0 : OksFile f(pf->get_full_file_name() + suffix, pf->get_logical_name(), pf->get_type(), pf->get_oks_format(), this);
3591 :
3592 0 : f.p_created_by = pf->p_created_by;
3593 0 : f.p_creation_time = pf->p_creation_time;
3594 0 : f.p_created_on = pf->p_created_on;
3595 0 : f.p_list_of_include_files = pf->p_list_of_include_files;
3596 :
3597 0 : if (!p_silence) {
3598 0 : std::lock_guard lock(p_parallel_out_mutex);
3599 0 : std::cout << "Making backup of data file \"" << pf->p_full_name << "\"...\n";
3600 0 : }
3601 :
3602 0 : bool silence = p_silence;
3603 :
3604 0 : p_silence = true;
3605 :
3606 0 : try {
3607 0 : k_save_data(&f, true, pf);
3608 0 : } catch (exception& ex) {
3609 0 : p_silence = silence;
3610 0 : throw CanNotBackupFile(pf->p_full_name, ex);
3611 0 : }
3612 :
3613 0 : p_silence = silence;
3614 0 : }
3615 :
3616 : // kernel method
3617 :
3618 : void
3619 0 : OksKernel::k_save_data(OksFile* pf,
3620 : bool ignoreBadObjects,
3621 : OksFile* fh,
3622 : const OksObject::FSet* objects,
3623 : bool force_defaults)
3624 : {
3625 0 : const char _fname[] = "k_save_data";
3626 0 : std::string fname = make_fname(_fname, sizeof(_fname) - 1, pf->p_full_name, nullptr, nullptr);
3627 :
3628 0 : OSK_PROFILING(OksProfiler::KernelSaveData, this)
3629 0 : OSK_VERBOSE_REPORT("ENTER " << fname)
3630 :
3631 0 : std::string tmp_file_name;
3632 :
3633 0 : if (!fh)
3634 0 : fh = pf;
3635 :
3636 0 : try {
3637 :
3638 : // calculate number of objects in file and check objects
3639 :
3640 0 : size_t numberOfObjects = 0;
3641 :
3642 0 : if (objects) {
3643 0 : numberOfObjects = objects->size();
3644 :
3645 0 : if (!ignoreBadObjects) {
3646 0 : std::string errors;
3647 :
3648 0 : for (OksObject::FSet::const_iterator i = objects->begin(); i != objects->end(); ++i) {
3649 0 : errors += (*i)->report_dangling_references();
3650 : }
3651 :
3652 0 : if (!errors.empty()) {
3653 0 : std::ostringstream text;
3654 0 : text << "the file contains objects with dangling references:\n" << errors;
3655 0 : throw std::runtime_error(text.str().c_str());
3656 0 : }
3657 0 : }
3658 : } else {
3659 : // get list of all includes
3660 :
3661 0 : std::set<OksFile*> includes;
3662 0 : pf->get_all_include_files(this, includes);
3663 :
3664 0 : OSK_VERBOSE_REPORT("Test consistency of objects in file \"" << pf->p_full_name << '\"')
3665 :
3666 0 : bool found_bad_object = false;
3667 :
3668 0 : for (OksObject::Set::iterator i = p_objects.begin(); i != p_objects.end(); ++i) {
3669 0 : if ((*i)->file == fh) {
3670 0 : numberOfObjects++;
3671 0 : if (!ignoreBadObjects || !p_silence) {
3672 0 : if ((*i)->is_consistent(includes, "WARNING") == false) {
3673 0 : found_bad_object = true;
3674 : }
3675 0 : if ((*i)->is_duplicated() == true) {
3676 0 : if (!p_silence) {
3677 0 : Oks::error_msg(fname) << " The file contains duplicated object " << *i << std::endl;
3678 : }
3679 : found_bad_object = true;
3680 : }
3681 : }
3682 : }
3683 : }
3684 :
3685 0 : if (found_bad_object && ignoreBadObjects == false) {
3686 0 : throw std::runtime_error("the file contains inconsistent/duplicated objects or misses includes");
3687 : }
3688 0 : }
3689 :
3690 : // lock the file, if it is not locked already
3691 :
3692 0 : if (pf->is_locked() == false) {
3693 0 : pf->lock();
3694 : }
3695 :
3696 : // check if it possible to open the non-existent file in write mode
3697 :
3698 0 : {
3699 : // check first if file already exists
3700 :
3701 0 : std::fstream f(pf->p_full_name.c_str(), std::ios::in);
3702 :
3703 0 : if (!f) {
3704 :
3705 : // no check for non-existent file
3706 :
3707 0 : std::fstream f2(pf->p_full_name.c_str(), std::ios::out);
3708 :
3709 0 : if (!f2) {
3710 0 : std::ostringstream text;
3711 0 : text << "cannot open file \'" << pf->p_full_name << "\' for writing";
3712 0 : throw std::runtime_error(text.str().c_str());
3713 0 : }
3714 0 : }
3715 0 : }
3716 :
3717 0 : tmp_file_name = get_tmp_file(pf->p_full_name);
3718 :
3719 0 : long file_len = 0;
3720 :
3721 0 : {
3722 0 : std::ofstream f(tmp_file_name.c_str());
3723 :
3724 0 : f.exceptions(std::ostream::failbit | std::ostream::badbit);
3725 :
3726 0 : if (!f) {
3727 0 : std::ostringstream text;
3728 0 : text << "cannot create temporal file \'" << tmp_file_name << "\' to save data";
3729 0 : throw std::runtime_error(text.str().c_str());
3730 0 : } else {
3731 0 : if (!p_silence) {
3732 0 : std::lock_guard lock(p_parallel_out_mutex);
3733 0 : std::cout << "Saving " << numberOfObjects << " objects "
3734 : << (force_defaults ? "with enforced default values " : "") << "to data file \"" << pf->p_full_name
3735 0 : << "\"...\n";
3736 0 : }
3737 : }
3738 :
3739 : // set header parameters
3740 :
3741 0 : OksXmlOutputStream xmls(f);
3742 :
3743 0 : pf->p_number_of_items = numberOfObjects;
3744 0 : pf->p_oks_format = "data";
3745 :
3746 0 : pf->write(xmls);
3747 :
3748 0 : if (!p_objects.empty()) {
3749 0 : for (OksClass::Map::const_iterator i = p_classes.begin(); i != p_classes.end() && f.good(); ++i) {
3750 0 : OksObject::SMap sorted;
3751 :
3752 0 : for (OksObject::Map::const_iterator j = i->second->p_objects->begin(); j != i->second->p_objects->end();
3753 0 : ++j) {
3754 0 : if (j->second->file == fh || (objects && (objects->find(j->second) != objects->end()))) {
3755 0 : sorted[j->first] = j->second;
3756 : }
3757 : }
3758 :
3759 0 : for (OksObject::SMap::iterator j = sorted.begin(); j != sorted.end(); ++j) {
3760 0 : j->second->put(xmls, force_defaults);
3761 0 : xmls.put_raw('\n');
3762 : }
3763 0 : }
3764 : }
3765 :
3766 0 : xmls.put_last_tag("oks-data", sizeof("oks-data") - 1);
3767 0 : file_len = f.tellp();
3768 :
3769 0 : f.close();
3770 0 : }
3771 :
3772 : // check that the written file is OK
3773 : // FIXME: can be removed later, if exceptions work well enough
3774 :
3775 0 : {
3776 0 : long written_len = 0;
3777 :
3778 0 : std::shared_ptr<std::ifstream> f(new std::ifstream(tmp_file_name.c_str()));
3779 :
3780 0 : if (*f) {
3781 0 : f->seekg(0, std::ios::end);
3782 0 : written_len = static_cast<std::streamoff>(f->tellg());
3783 : }
3784 :
3785 0 : if (written_len != file_len) {
3786 0 : unlink(tmp_file_name.c_str());
3787 0 : std::ostringstream text;
3788 0 : text << "write error in file \'" << tmp_file_name << "\': " << written_len << " bytes been written instead of "
3789 0 : << file_len;
3790 0 : throw std::runtime_error(text.str().c_str());
3791 0 : }
3792 0 : }
3793 :
3794 : // remember file's mode
3795 :
3796 0 : struct stat buf;
3797 0 : if (int code = stat(pf->p_full_name.c_str(), &buf)) {
3798 0 : std::ostringstream text;
3799 0 : text << "cannot get information about file \'" << pf->p_full_name << "\': stat() failed with code " << code
3800 0 : << ", reason = \'" << strerror(errno) << '\'';
3801 0 : throw std::runtime_error(text.str().c_str());
3802 0 : }
3803 :
3804 : // rename temporal file
3805 :
3806 0 : if (int code = rename(tmp_file_name.c_str(), pf->p_full_name.c_str())) {
3807 0 : std::ostringstream text;
3808 0 : text << "cannot rename file \'" << tmp_file_name << "\' to \'" << pf->p_full_name
3809 0 : << "\': rename() failed with code " << code << ", reason = \'" << strerror(errno) << '\'';
3810 0 : throw std::runtime_error(text.str().c_str());
3811 0 : }
3812 :
3813 0 : tmp_file_name.erase(0);
3814 :
3815 0 : if (pf != p_active_data) {
3816 0 : try {
3817 0 : pf->unlock();
3818 0 : } catch (exception& ex) {
3819 0 : throw std::runtime_error(ex.what());
3820 0 : }
3821 : }
3822 0 : pf->p_is_updated = false;
3823 0 : pf->update_status_of_file();
3824 :
3825 : // get mode for new file
3826 :
3827 0 : struct stat buf2;
3828 0 : if (int code = stat(pf->p_full_name.c_str(), &buf2)) {
3829 0 : std::ostringstream text;
3830 0 : text << "cannot get information about file \'" << pf->p_full_name << "\': stat() failed with code " << code
3831 0 : << ", reason = \'" << strerror(errno) << '\'';
3832 0 : throw std::runtime_error(text.str().c_str());
3833 0 : }
3834 :
3835 : // set file's mode if needed
3836 :
3837 0 : if (buf.st_mode != buf2.st_mode) {
3838 0 : if (int code = chmod(pf->p_full_name.c_str(), buf.st_mode) != 0) {
3839 0 : std::ostringstream text;
3840 0 : text << "cannot set protection mode for file \'" << pf->p_full_name << "\': chmod() failed with code " << code
3841 0 : << ", reason = \'" << strerror(errno) << '\'';
3842 0 : throw std::runtime_error(text.str().c_str());
3843 0 : }
3844 : }
3845 :
3846 : // set file's group if needed (in case of problems report error, but do not throw exception)
3847 :
3848 0 : if (buf.st_gid != buf2.st_gid) {
3849 0 : if (int code = chown(pf->p_full_name.c_str(), (gid_t)(-1), buf.st_gid) != 0) {
3850 0 : ers::warning(kernel::SetGroupIdFailed(ERS_HERE, buf.st_gid, pf->p_full_name.c_str(), code, strerror(errno)));
3851 : }
3852 : }
3853 :
3854 : }
3855 :
3856 0 : catch (exception& e) {
3857 0 : if (pf != p_active_data) {
3858 0 : try {
3859 0 : pf->unlock();
3860 0 : } catch (...) {
3861 0 : }
3862 : }
3863 0 : if (!tmp_file_name.empty()) {
3864 0 : unlink(tmp_file_name.c_str());
3865 : }
3866 0 : throw CanNotWriteToFile("k_save_data", "data file", pf->p_full_name, e);
3867 0 : } catch (std::exception& e) {
3868 0 : if (pf != p_active_data) {
3869 0 : try {
3870 0 : pf->unlock();
3871 0 : } catch (...) {
3872 0 : }
3873 : }
3874 0 : if (!tmp_file_name.empty()) {
3875 0 : unlink(tmp_file_name.c_str());
3876 : }
3877 0 : throw CanNotWriteToFile("k_save_data", "data file", pf->p_full_name, e.what());
3878 0 : } catch (...) {
3879 0 : if (pf != p_active_data) {
3880 0 : try {
3881 0 : pf->unlock();
3882 0 : } catch (...) {
3883 0 : }
3884 : }
3885 0 : if (!tmp_file_name.empty()) {
3886 0 : unlink(tmp_file_name.c_str());
3887 : }
3888 0 : throw CanNotWriteToFile("k_save_data", "data file", pf->p_full_name, "unknown");
3889 0 : }
3890 :
3891 0 : OSK_VERBOSE_REPORT("LEAVE " << fname)
3892 0 : }
3893 :
3894 : // note: the file name is used as a key in the map
3895 : // to rename a file it is necessary to remove file from map, change name and insert it back
3896 :
3897 : void
3898 0 : OksKernel::k_rename_data(OksFile* pf, const std::string& short_name, const std::string& long_name)
3899 : {
3900 0 : remove_data_file(pf);
3901 0 : pf->rename(short_name, long_name);
3902 0 : add_data_file(pf);
3903 0 : }
3904 :
3905 : void
3906 0 : OksKernel::save_as_data(const std::string& new_name, OksFile* pf)
3907 : {
3908 0 : const char _fname[] = "save_as_data";
3909 0 : std::string fname = make_fname(_fname, sizeof(_fname) - 1, new_name, nullptr, &pf);
3910 0 : OSK_VERBOSE_REPORT("ENTER " << fname)
3911 :
3912 0 : try {
3913 :
3914 0 : if (!new_name.length()) {
3915 0 : throw std::runtime_error("the new filename is empty");
3916 : }
3917 :
3918 0 : std::unique_lock lock(p_kernel_mutex);
3919 :
3920 0 : std::string old_short_name = pf->p_short_name;
3921 0 : std::string old_full_name = pf->p_full_name;
3922 0 : std::string old_format = pf->p_oks_format;
3923 :
3924 0 : k_rename_data(pf, new_name, new_name);
3925 :
3926 0 : try {
3927 0 : k_save_data(pf);
3928 0 : } catch (...) {
3929 0 : k_rename_data(pf, old_short_name, old_full_name);
3930 0 : pf->p_oks_format = old_format;
3931 0 : throw;
3932 0 : }
3933 :
3934 0 : } catch (exception& ex) {
3935 0 : throw CanNotWriteToFile("k_save_as_data", "data file", new_name, ex);
3936 0 : } catch (std::exception& ex) {
3937 0 : throw CanNotWriteToFile("k_save_as_data", "data file", new_name, ex.what());
3938 0 : }
3939 :
3940 0 : OSK_VERBOSE_REPORT("LEAVE " << fname)
3941 0 : }
3942 :
3943 : void
3944 0 : OksKernel::save_all_data(bool force_defaults)
3945 : {
3946 0 : TLOG_DEBUG(4) << "enter";
3947 :
3948 0 : {
3949 0 : std::shared_lock lock(p_kernel_mutex);
3950 :
3951 0 : for (OksFile::Map::iterator i = p_data_files.begin(); i != p_data_files.end(); ++i) {
3952 0 : if (check_read_only(i->second) == false) {
3953 0 : k_save_data(i->second, false, nullptr, nullptr, force_defaults);
3954 : } else {
3955 0 : TLOG_DEBUG(2) << "skip read-only data file \'" << *(i->first) << '\'';
3956 : }
3957 : }
3958 0 : }
3959 :
3960 0 : TLOG_DEBUG(4) << "exit";
3961 0 : }
3962 :
3963 : // user-allowed method
3964 :
3965 : void
3966 0 : OksKernel::close_data(OksFile* fp, bool unbind)
3967 : {
3968 0 : std::unique_lock lock(p_kernel_mutex);
3969 0 : k_close_data(fp, unbind);
3970 0 : }
3971 :
3972 : // kernel method
3973 :
3974 : void
3975 138 : OksKernel::k_close_data(OksFile* fp, bool unbind)
3976 : {
3977 138 : const char _fname[] = "k_close_data";
3978 138 : std::string fname = make_fname(_fname, sizeof(_fname) - 1, fp->p_full_name, &unbind, nullptr);
3979 :
3980 138 : OSK_PROFILING(OksProfiler::KernelCloseData, this)
3981 138 : OSK_VERBOSE_REPORT("ENTER " << fname)
3982 :
3983 138 : try {
3984 138 : fp->unlock();
3985 0 : } catch (exception& ex) {
3986 0 : Oks::error_msg(fname) << ex.what() << std::endl;
3987 0 : }
3988 :
3989 138 : if (!p_silence) {
3990 0 : std::lock_guard lock(p_parallel_out_mutex);
3991 0 : std::cout << (fp->p_included_by ? " * c" : "C") << "lose OKS data \"" << fp->p_full_name << "\"..." << std::endl;
3992 0 : }
3993 :
3994 138 : if (unbind) {
3995 0 : for (OksObject::Set::const_iterator i = p_objects.begin(); i != p_objects.end(); ++i) {
3996 0 : OksObject* o = *i;
3997 0 : if (o->file != fp)
3998 0 : o->unbind_file(fp);
3999 : }
4000 : }
4001 :
4002 138 : if (p_close_all == false) {
4003 0 : std::list<OksObject*>* olist = create_list_of_data_objects(fp);
4004 :
4005 0 : if (olist) {
4006 0 : while (!olist->empty()) {
4007 0 : OksObject* o = olist->front();
4008 0 : olist->pop_front();
4009 0 : if (!is_dangling(o))
4010 0 : delete o;
4011 : }
4012 :
4013 0 : delete olist;
4014 : }
4015 : }
4016 :
4017 138 : remove_data_file(fp);
4018 138 : delete fp;
4019 :
4020 138 : if (p_active_data == fp)
4021 5 : p_active_data = 0;
4022 :
4023 138 : OSK_VERBOSE_REPORT("LEAVE " << fname)
4024 138 : }
4025 :
4026 : void
4027 190 : OksKernel::close_all_data()
4028 : {
4029 190 : TLOG_DEBUG(4) << "enter";
4030 :
4031 190 : {
4032 190 : std::unique_lock lock(p_kernel_mutex);
4033 :
4034 190 : p_close_all = true;
4035 :
4036 328 : while (!p_data_files.empty()) {
4037 138 : k_close_data(p_data_files.begin()->second, false);
4038 : }
4039 :
4040 190 : if (!p_objects.empty()) {
4041 5566 : for (OksObject::Set::iterator i = p_objects.begin(); i != p_objects.end(); ++i) {
4042 5471 : delete *i;
4043 : }
4044 :
4045 95 : p_objects.clear();
4046 :
4047 7597 : for (OksClass::Map::const_iterator i = p_classes.begin(); i != p_classes.end(); ++i) {
4048 7502 : if (i->second->p_objects) {
4049 7502 : delete i->second->p_objects;
4050 7502 : i->second->p_objects = 0;
4051 : }
4052 : }
4053 : }
4054 :
4055 190 : p_close_all = false;
4056 190 : }
4057 :
4058 190 : TLOG_DEBUG(4) << "exit";
4059 190 : }
4060 :
4061 : void
4062 27 : OksKernel::set_active_data(OksFile* fp)
4063 : {
4064 27 : std::unique_lock lock(p_kernel_mutex);
4065 27 : k_set_active_data(fp);
4066 27 : }
4067 :
4068 : void
4069 32 : OksKernel::k_set_active_data(OksFile* fp)
4070 : {
4071 32 : TLOG_DEBUG(4) << "enter for file " << (void*)fp;
4072 :
4073 : // check if active data is different from given file
4074 :
4075 32 : if (p_active_data == fp)
4076 : return;
4077 :
4078 : // first unlock current active data
4079 :
4080 5 : if (p_active_data && p_active_data->is_updated() == false) {
4081 0 : try {
4082 0 : p_active_data->unlock();
4083 0 : } catch (exception& ex) {
4084 0 : throw CanNotSetActiveFile("data", fp->get_full_file_name(), ex);
4085 0 : }
4086 : }
4087 :
4088 : // exit, if file is (null)
4089 :
4090 5 : if (!fp) {
4091 0 : p_active_data = 0;
4092 0 : return;
4093 : }
4094 :
4095 5 : try {
4096 5 : fp->lock();
4097 5 : p_active_data = fp;
4098 0 : } catch (exception& ex) {
4099 0 : throw CanNotSetActiveFile("data", fp->get_full_file_name(), ex);
4100 0 : }
4101 :
4102 5 : TLOG_DEBUG(4) << "leave for file " << (void*)fp;
4103 : }
4104 :
4105 : std::list<OksObject*>*
4106 0 : OksKernel::create_list_of_data_objects(OksFile* fp) const
4107 : {
4108 0 : const char _fname[] = "create_list_of_data_objects";
4109 0 : std::string fname = make_fname(_fname, sizeof(_fname) - 1, fp->get_full_file_name(), nullptr, nullptr);
4110 :
4111 0 : OSK_PROFILING(OksProfiler::KernelCreateDataObjectList, this)
4112 0 : OSK_VERBOSE_REPORT("ENTER " << fname)
4113 :
4114 0 : std::list<OksObject*>* olist = new std::list<OksObject*>();
4115 :
4116 0 : if (!p_objects.empty()) {
4117 0 : for (OksObject::Set::const_iterator i = p_objects.begin(); i != p_objects.end(); ++i) {
4118 0 : if ((*i)->file == fp)
4119 0 : olist->push_back(*i);
4120 : }
4121 : }
4122 :
4123 0 : if (olist->empty()) {
4124 0 : delete olist;
4125 0 : olist = 0;
4126 : }
4127 :
4128 0 : OSK_VERBOSE_REPORT("LEAVE " << fname)
4129 :
4130 0 : return olist;
4131 0 : }
4132 :
4133 : /******************************************************************************/
4134 :
4135 : // user-allowed method
4136 :
4137 : void
4138 5 : OksKernel::bind_objects()
4139 : {
4140 5 : std::unique_lock lock(p_kernel_mutex);
4141 5 : k_bind_objects();
4142 5 : }
4143 :
4144 : void
4145 95 : OksKernel::k_bind_objects()
4146 : {
4147 95 : TLOG_DEBUG(4) << "enter";
4148 :
4149 95 : p_bind_objects_status.clear();
4150 :
4151 95 : if (!p_objects.empty()) {
4152 5534 : for (OksObject::Set::iterator i = p_objects.begin(); i != p_objects.end(); ++i) {
4153 5444 : try {
4154 5444 : (*i)->bind_objects();
4155 106 : } catch (ObjectBindError& ex) {
4156 106 : if (ex.p_is_error) {
4157 0 : throw;
4158 : } else {
4159 106 : const std::string error_text(strchr(ex.what(), '\n') + 1);
4160 106 : if (!p_bind_objects_status.empty())
4161 89 : p_bind_objects_status.push_back('\n');
4162 106 : p_bind_objects_status.append(error_text);
4163 :
4164 106 : TLOG_DEBUG(1) << error_text;
4165 106 : }
4166 106 : }
4167 : }
4168 : }
4169 :
4170 95 : if (!p_silence && !p_bind_objects_status.empty()) {
4171 0 : ers::warning(kernel::BindError(ERS_HERE, p_bind_objects_status));
4172 : }
4173 :
4174 95 : TLOG_DEBUG(4) << "exit with status:\n" << p_bind_objects_status;
4175 95 : }
4176 :
4177 : void
4178 0 : OksKernel::unbind_all_rels(const OksObject::FSet& rm_objs, OksObject::FSet& updated) const
4179 : {
4180 0 : const OksClass::Map& all_classes(classes());
4181 :
4182 0 : for (OksClass::Map::const_iterator i = all_classes.begin(); i != all_classes.end(); ++i) {
4183 0 : OksClass* c(i->second);
4184 0 : if (const OksObject::Map* objs = i->second->objects()) {
4185 0 : for (OksObject::Map::const_iterator j = objs->begin(); j != objs->end(); ++j) {
4186 0 : OksObject* o(j->second);
4187 0 : unsigned short l1 = c->number_of_all_attributes();
4188 0 : unsigned short l2 = l1 + c->number_of_all_relationships();
4189 :
4190 0 : while (l1 < l2) {
4191 0 : OksDataInfo odi(l1, (OksRelationship*)0);
4192 0 : OksData* d(o->GetRelationshipValue(&odi));
4193 :
4194 0 : if (d->type == OksData::object_type) {
4195 0 : if (rm_objs.find(d->data.OBJECT) != rm_objs.end()) {
4196 0 : updated.insert(o);
4197 0 : const OksClass* __c(d->data.OBJECT->GetClass());
4198 0 : const OksString& __id(d->data.OBJECT->GetId());
4199 0 : d->Set(__c, __id);
4200 0 : TLOG_DEBUG(5) << "set relationship of " << o << ": " << *d;
4201 0 : }
4202 0 : } else if (d->type == OksData::list_type) {
4203 0 : for (OksData::List::const_iterator li = d->data.LIST->begin(); li != d->data.LIST->end(); ++li) {
4204 0 : OksData* lid(*li);
4205 0 : if (lid->type == OksData::object_type) {
4206 0 : if (rm_objs.find(lid->data.OBJECT) != rm_objs.end()) {
4207 0 : updated.insert(o);
4208 0 : const OksClass* __c(lid->data.OBJECT->GetClass());
4209 0 : const OksString& __id(lid->data.OBJECT->GetId());
4210 0 : lid->Set(__c, __id);
4211 0 : TLOG_DEBUG(5) << "set relationship of " << o << ": " << *d;
4212 0 : }
4213 : }
4214 : }
4215 : }
4216 0 : ++l1;
4217 : }
4218 : }
4219 : }
4220 : }
4221 0 : }
4222 :
4223 : OksClass*
4224 35353 : OksKernel::find_class(const char* name) const
4225 : {
4226 35353 : OksClass::Map::const_iterator i = p_classes.find(name);
4227 35353 : return (i != p_classes.end() ? i->second : 0);
4228 : }
4229 :
4230 : void
4231 0 : OksKernel::k_add(OksClass* c)
4232 : {
4233 0 : TLOG_DEBUG(4) << "enter for class \"" << c->get_name() << '\"';
4234 :
4235 0 : if (p_active_schema == 0) {
4236 0 : throw CannotAddClass(*c, "no active schema");
4237 : }
4238 :
4239 0 : if (p_classes.find(c->get_name().c_str()) != p_classes.end()) {
4240 0 : throw CannotAddClass(*c, "class already exists");
4241 : }
4242 :
4243 0 : c->p_kernel = this;
4244 0 : c->p_file = p_active_schema;
4245 0 : c->p_file->p_is_updated = true;
4246 :
4247 0 : p_classes[c->get_name().c_str()] = c;
4248 :
4249 0 : try {
4250 0 : registrate_all_classes(false);
4251 0 : } catch (exception& ex) {
4252 0 : throw CannotAddClass(*c, ex);
4253 0 : }
4254 :
4255 0 : if (OksClass::create_notify_fn)
4256 0 : (*OksClass::create_notify_fn)(c);
4257 0 : }
4258 :
4259 : void
4260 7502 : OksKernel::k_remove(OksClass* c)
4261 : {
4262 7502 : TLOG_DEBUG(4) << "enter for class \"" << c->get_name() << '\"';
4263 :
4264 7502 : if (OksClass::delete_notify_fn)
4265 0 : (*OksClass::delete_notify_fn)(c);
4266 :
4267 7502 : p_classes.erase(c->get_name().c_str());
4268 7502 : }
4269 :
4270 : void
4271 187 : OksKernel::registrate_all_classes(bool skip_registered)
4272 : {
4273 187 : std::unique_lock lock(p_schema_mutex); // protect schema and all objects from changes
4274 :
4275 187 : if (size_t num_of_classes = p_classes.size()) {
4276 187 : OksClass** table = new OksClass*[num_of_classes];
4277 187 : size_t array_size =
4278 : num_of_classes * sizeof(OksClass*) / sizeof(wchar_t); // is used by wmemset() to init above array with NULLs
4279 :
4280 187 : OksClass::Map::iterator i = p_classes.begin();
4281 187 : unsigned long idx(0);
4282 :
4283 14226 : for (; i != p_classes.end(); ++i)
4284 14039 : i->second->registrate_class(skip_registered);
4285 :
4286 : // create lists of sub-classes
4287 : // note: do not use unscalable OksClass::create_sub_classes()
4288 14226 : for (i = p_classes.begin(); i != p_classes.end(); ++i) {
4289 14039 : OksClass* c(i->second);
4290 :
4291 14039 : if (!c->p_all_sub_classes)
4292 7502 : c->p_all_sub_classes = new OksClass::FList();
4293 : else
4294 6537 : c->p_all_sub_classes->clear();
4295 :
4296 14039 : c->p_id = idx++;
4297 : }
4298 :
4299 14226 : for (i = p_classes.begin(); i != p_classes.end(); ++i) {
4300 14039 : OksClass* c(i->second);
4301 14039 : if (const OksClass::FList* scl = c->p_all_super_classes) {
4302 27217 : for (OksClass::FList::const_iterator j = scl->begin(); j != scl->end(); ++j) {
4303 13178 : (*j)->p_all_sub_classes->push_back(c);
4304 : }
4305 : }
4306 : }
4307 :
4308 187 : if (get_test_duplicated_objects_via_inheritance_mode() && !get_allow_duplicated_objects_mode()) {
4309 295 : for (i = p_classes.begin(); i != p_classes.end(); ++i) {
4310 290 : OksClass* c(i->second);
4311 290 : if (!c->get_is_abstract()) {
4312 215 : if (const OksClass::FList* spc = c->all_super_classes()) {
4313 215 : if (!spc->empty()) {
4314 130 : wmemset(reinterpret_cast<wchar_t*>(table), 0, array_size); // [re-]set table by NULLs
4315 330 : for (OksClass::FList::const_iterator j1 = spc->begin(); j1 != spc->end(); ++j1) {
4316 200 : if (const OksClass::FList* sbc = (*j1)->all_sub_classes()) {
4317 1640 : for (OksClass::FList::const_iterator j2 = sbc->begin(); j2 != sbc->end(); ++j2) {
4318 1440 : OksClass* c2(*j2);
4319 1440 : if ((c2 != c) && !c2->get_is_abstract()) {
4320 800 : table[c2->p_id] = c2;
4321 : }
4322 : }
4323 : }
4324 : }
4325 :
4326 130 : unsigned int count(0);
4327 7670 : for (unsigned int x = 0; x < num_of_classes; ++x) {
4328 7540 : if (table[x])
4329 690 : count++;
4330 : }
4331 :
4332 130 : if (count) {
4333 120 : std::vector<OksClass*>* cih;
4334 120 : if (!c->p_inheritance_hierarchy) {
4335 120 : cih = c->p_inheritance_hierarchy = new std::vector<OksClass*>();
4336 : } else {
4337 0 : cih = c->p_inheritance_hierarchy;
4338 0 : cih->clear();
4339 : }
4340 :
4341 120 : cih->reserve(count);
4342 :
4343 7080 : for (unsigned int x = 0; x < num_of_classes; ++x) {
4344 6960 : if (table[x])
4345 690 : cih->push_back(table[x]);
4346 : }
4347 : }
4348 : }
4349 : }
4350 : }
4351 : }
4352 :
4353 : #ifndef ERS_NO_DEBUG
4354 5 : if (ers::debug_level() >= 2) {
4355 3 : std::ostringstream s;
4356 :
4357 177 : for (i = p_classes.begin(); i != p_classes.end(); ++i) {
4358 174 : if (std::vector<OksClass*>* cis = i->second->p_inheritance_hierarchy) {
4359 72 : s << " - class \'" << i->second->get_name() << "\' shares IDs with " << cis->size() << " classes: ";
4360 72 : OksClass::Set sorted;
4361 486 : for (std::vector<OksClass*>::const_iterator j = cis->begin(); j != cis->end(); ++j) {
4362 414 : sorted.insert(*j);
4363 : }
4364 486 : for (OksClass::Set::const_iterator j = sorted.begin(); j != sorted.end(); ++j) {
4365 414 : if (j != sorted.begin())
4366 342 : s << ", ";
4367 414 : s << '\'' << (*j)->get_name() << '\'';
4368 : }
4369 72 : s << std::endl;
4370 72 : }
4371 : }
4372 :
4373 3 : TLOG_DEBUG(2) << "Schema inheritance hierarchy used to test objects with equal IDs:\n" << s.str();
4374 3 : }
4375 : #endif
4376 : }
4377 :
4378 187 : delete[] table;
4379 :
4380 187 : k_check_bind_classes_status();
4381 : }
4382 187 : }
4383 :
4384 : bool
4385 0 : OksKernel::is_dangling(OksObject* o) const
4386 : {
4387 0 : OSK_PROFILING(OksProfiler::KernelIsDanglingObject, this)
4388 0 : OSK_VERBOSE_REPORT("Enter OksKernel::is_dangling(OksObject *)")
4389 :
4390 0 : bool not_found;
4391 :
4392 0 : {
4393 0 : std::lock_guard lock(p_objects_mutex);
4394 0 : not_found = (p_objects.find(o) == p_objects.end());
4395 0 : }
4396 :
4397 0 : if (p_verbose) {
4398 0 : std::string fname("OksKernel::is_dangling(");
4399 :
4400 0 : if (not_found)
4401 0 : fname += "?object?) returns true";
4402 : else {
4403 0 : fname += '\"';
4404 0 : fname += o->GetId();
4405 0 : fname += "\") returns false";
4406 : }
4407 :
4408 0 : OSK_VERBOSE_REPORT("LEAVE " << fname.c_str())
4409 0 : }
4410 :
4411 0 : return not_found;
4412 0 : }
4413 :
4414 : bool
4415 15076 : OksKernel::is_dangling(OksClass* c) const
4416 : {
4417 15076 : OSK_PROFILING(OksProfiler::KernelIsDanglingClass, this)
4418 15076 : OSK_VERBOSE_REPORT("Enter OksKernel::is_dangling(OksClass *)")
4419 :
4420 15076 : bool not_found = true;
4421 :
4422 15076 : {
4423 15076 : if (!p_classes.empty()) {
4424 441113 : for (OksClass::Map::const_iterator i = p_classes.begin(); i != p_classes.end(); ++i) {
4425 433766 : if (i->second == c) {
4426 : not_found = false;
4427 : break;
4428 : }
4429 : }
4430 : }
4431 : }
4432 :
4433 15076 : if (p_verbose) {
4434 0 : std::string fname("OksKernel::is_dangling(");
4435 :
4436 0 : if (not_found)
4437 0 : fname += "?class?) returns true";
4438 : else {
4439 0 : fname += '\"';
4440 0 : fname += c->get_name();
4441 0 : fname += "\") returns false";
4442 : }
4443 :
4444 0 : OSK_VERBOSE_REPORT("LEAVE " << fname.c_str())
4445 0 : }
4446 :
4447 15076 : return not_found;
4448 15076 : }
4449 :
4450 : void
4451 0 : OksKernel::get_all_classes(const std::vector<std::string>& names_in, ClassSet& classes_out) const
4452 : {
4453 0 : for (std::vector<std::string>::const_iterator i = names_in.begin(); i != names_in.end(); ++i) {
4454 0 : OksClass* c = find_class(*i);
4455 0 : if (c != 0 && classes_out.find(c) == classes_out.end()) {
4456 0 : classes_out.insert(c);
4457 0 : if (const OksClass::FList* sc = c->all_sub_classes()) {
4458 0 : for (OksClass::FList::const_iterator j = sc->begin(); j != sc->end(); ++j) {
4459 0 : classes_out.insert(*j);
4460 : }
4461 : }
4462 : }
4463 : }
4464 0 : }
4465 :
4466 : // keep information about repository root
4467 : // throw exception if TDAQ_DB_REPOSITORY or TDAQ_DB_USER_REPOSITORY are not defined
4468 :
4469 : struct ReposDirs
4470 : {
4471 :
4472 : ReposDirs(const char* op, const char* cwd, OksKernel* kernel);
4473 : ~ReposDirs();
4474 :
4475 : const char* p_repository_root;
4476 : const char* p_user_repository_root;
4477 : size_t p_repos_dir_len;
4478 : size_t p_user_dir_len;
4479 : const char* p_dir;
4480 :
4481 : static std::mutex p_mutex;
4482 : };
4483 :
4484 : std::mutex ReposDirs::p_mutex;
4485 :
4486 0 : ReposDirs::ReposDirs(const char* op, const char* cwd, OksKernel* kernel)
4487 : {
4488 0 : p_repository_root = OksKernel::get_repository_root().c_str();
4489 0 : if (OksKernel::get_repository_root().empty()) {
4490 0 : throw RepositoryOperationFailed(op,
4491 0 : "the repository-root is not set (check environment variable TDAQ_DB_REPOSITORY)");
4492 : } else {
4493 0 : p_repos_dir_len = strlen(p_repository_root);
4494 : }
4495 :
4496 0 : p_user_repository_root = kernel->get_user_repository_root().c_str();
4497 0 : if (!*p_user_repository_root) {
4498 0 : throw RepositoryOperationFailed(
4499 0 : op, "the user-repository-root is not set (check environment variable TDAQ_DB_USER_REPOSITORY)");
4500 : } else {
4501 0 : p_user_dir_len = strlen(p_user_repository_root);
4502 : }
4503 :
4504 0 : p_mutex.lock();
4505 :
4506 0 : if (strcmp(cwd, p_user_repository_root)) {
4507 0 : if (int result = chdir(p_user_repository_root)) {
4508 0 : p_mutex.unlock();
4509 0 : std::ostringstream text;
4510 0 : text << "chdir (\'" << p_user_repository_root << "\') failed with code " << result << ": " << strerror(errno);
4511 0 : throw RepositoryOperationFailed(op, text.str());
4512 0 : }
4513 0 : p_dir = cwd;
4514 0 : TLOG_DEBUG(2) << "change cwd: \"" << p_user_repository_root << '\"';
4515 : } else {
4516 0 : p_dir = 0;
4517 0 : TLOG_DEBUG(2) << "cwd: \"" << p_user_repository_root << "\" is equal to target dir";
4518 : }
4519 0 : }
4520 :
4521 0 : ReposDirs::~ReposDirs()
4522 : {
4523 0 : if (p_dir) {
4524 0 : if (int result = chdir(p_dir)) {
4525 0 : Oks::error_msg("ReposDirs::~ReposDirs()")
4526 0 : << "chdir (\'" << p_dir << "\') failed with code " << result << ": " << strerror(errno) << std::endl;
4527 : }
4528 0 : TLOG_DEBUG(2) << "restore cwd: \"" << p_dir << '\"';
4529 : }
4530 :
4531 0 : p_mutex.unlock();
4532 0 : }
4533 :
4534 : // keep information and delete output file on exit
4535 :
4536 : struct CommandOutput
4537 : {
4538 :
4539 : CommandOutput(const char* command_name, const OksKernel* k, std::string& cmd);
4540 : ~CommandOutput();
4541 :
4542 : const std::string& file_name() const { return p_file_name; }
4543 : std::string cat2str() const;
4544 : std::string last_str() const;
4545 :
4546 : void check_command_status(int status);
4547 :
4548 : std::string p_file_name;
4549 : std::string& p_command;
4550 : const char* p_command_name;
4551 : };
4552 :
4553 0 : CommandOutput::CommandOutput(const char* command_name, const OksKernel* k, std::string& cmd)
4554 0 : : p_command(cmd)
4555 0 : , p_command_name(command_name)
4556 : {
4557 0 : p_file_name = get_temporary_dir() + '/' + command_name + '.' + std::to_string(getpid()) + ':' +
4558 0 : std::to_string(reinterpret_cast<std::uintptr_t>(k)) + ".txt";
4559 :
4560 0 : p_command.append(" &>");
4561 0 : p_command.append(p_file_name);
4562 0 : }
4563 :
4564 0 : CommandOutput::~CommandOutput()
4565 : {
4566 0 : unlink(p_file_name.c_str());
4567 0 : }
4568 :
4569 : std::string
4570 0 : CommandOutput::cat2str() const
4571 : {
4572 0 : std::ifstream f(p_file_name.c_str());
4573 0 : std::string out;
4574 :
4575 0 : if (f) {
4576 0 : std::filebuf* buf = f.rdbuf();
4577 0 : while (true) {
4578 0 : char c = buf->sbumpc();
4579 0 : if (c == EOF)
4580 : break;
4581 : else
4582 0 : out += c;
4583 : }
4584 : }
4585 :
4586 0 : return out;
4587 0 : }
4588 :
4589 : std::string
4590 0 : CommandOutput::last_str() const
4591 : {
4592 0 : std::ifstream f(p_file_name.c_str());
4593 0 : std::string lastline;
4594 :
4595 0 : if (f.is_open()) {
4596 0 : f.seekg(-1, std::ios_base::end);
4597 0 : if (f.peek() == '\n') {
4598 0 : f.seekg(-1, std::ios_base::cur);
4599 0 : for (int i = f.tellg(); i >= 0; i--) {
4600 0 : if (f.peek() == '\n') {
4601 0 : f.get();
4602 : break;
4603 : }
4604 0 : f.seekg(i, std::ios_base::beg);
4605 : }
4606 : }
4607 :
4608 0 : getline(f, lastline);
4609 : }
4610 0 : return lastline;
4611 0 : }
4612 :
4613 : void
4614 0 : CommandOutput::check_command_status(int status)
4615 : {
4616 0 : if (status == -1) {
4617 0 : std::ostringstream text;
4618 0 : text << "cannot execute command \'" << p_command << "\': " << strerror(errno);
4619 0 : throw RepositoryOperationFailed(p_command_name, text.str());
4620 0 : } else if (int error_code = WEXITSTATUS(status)) {
4621 0 : std::ostringstream text;
4622 0 : text << p_command.substr(0, p_command.find_first_of(' ')) << " exit with error code " << error_code << ", output:\n"
4623 0 : << cat2str();
4624 0 : throw RepositoryOperationFailed(p_command_name, text.str());
4625 0 : }
4626 0 : }
4627 :
4628 : void
4629 0 : OksKernel::k_rename_repository_file(OksFile* file_h, const std::string& new_name)
4630 : {
4631 0 : if (file_h->p_oks_format == "schema") {
4632 0 : remove_schema_file(file_h);
4633 0 : file_h->rename(new_name);
4634 0 : add_schema_file(file_h);
4635 : } else {
4636 0 : remove_data_file(file_h);
4637 0 : file_h->rename(new_name);
4638 0 : add_data_file(file_h);
4639 : }
4640 0 : }
4641 :
4642 : void
4643 0 : OksKernel::k_checkout_repository(const std::string& param, const std::string& val, const std::string& branch_name)
4644 : {
4645 0 : if (get_repository_root().empty())
4646 0 : throw RepositoryOperationFailed("checkout", "the repository root is not set");
4647 :
4648 0 : ReposDirs reps("checkout", s_cwd, this);
4649 :
4650 0 : std::string cmd("oks-checkout.sh");
4651 :
4652 0 : if (get_verbose_mode()) {
4653 0 : cmd += " -v";
4654 : }
4655 :
4656 0 : cmd.append(" -u ");
4657 0 : cmd.append(get_user_repository_root());
4658 :
4659 0 : if (!param.empty()) {
4660 0 : cmd.append(" --");
4661 0 : cmd.append(param);
4662 0 : cmd.push_back(' ');
4663 0 : cmd.push_back('\"');
4664 0 : cmd.append(val);
4665 0 : cmd.push_back('\"');
4666 : }
4667 :
4668 0 : if (!branch_name.empty()) {
4669 0 : cmd.append(" -b ");
4670 0 : cmd.append(branch_name);
4671 : }
4672 :
4673 0 : auto start_usage = std::chrono::steady_clock::now();
4674 :
4675 0 : if (!p_silence) {
4676 0 : std::lock_guard lock(p_parallel_out_mutex);
4677 0 : log_timestamp() << "[OKS checkout] => " << cmd << std::endl;
4678 0 : }
4679 :
4680 0 : CommandOutput cmd_out("oks-checkout", this, cmd);
4681 0 : cmd_out.check_command_status(system(cmd.c_str()));
4682 :
4683 0 : p_repository_checkout_ts = p_repository_update_ts = std::time(0);
4684 :
4685 0 : if (!p_silence) {
4686 0 : std::lock_guard lock(p_parallel_out_mutex);
4687 0 : std::cout << cmd_out.cat2str();
4688 0 : log_timestamp()
4689 0 : << "[OKS checkout] => done in "
4690 0 : << std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - start_usage).count() /
4691 0 : 1000.
4692 0 : << " ms" << std::endl;
4693 0 : }
4694 :
4695 0 : static std::string version_prefix("checkout oks version ");
4696 0 : std::string version = cmd_out.last_str();
4697 0 : std::string::size_type pos = version.find(version_prefix);
4698 :
4699 0 : if (pos == 0)
4700 0 : p_repository_version = version.substr(version_prefix.size());
4701 : else
4702 0 : throw RepositoryOperationFailed("checkout", "cannot read oks version");
4703 0 : }
4704 :
4705 : void
4706 95 : OksKernel::remove_user_repository_dir()
4707 : {
4708 95 : if (p_allow_repository && p_user_repository_root_created) {
4709 0 : s_git_folders.remove(p_user_repository_root);
4710 0 : s_git_folders.erase(p_user_repository_root);
4711 0 : p_user_repository_root_created = false;
4712 : }
4713 95 : }
4714 :
4715 : void
4716 0 : OksKernel::unset_repository_created()
4717 : {
4718 0 : s_git_folders.erase(p_user_repository_root);
4719 0 : p_user_repository_root_created = false;
4720 0 : }
4721 :
4722 : void
4723 0 : OksKernel::k_copy_repository(const std::string& source, const std::string& destination)
4724 : {
4725 0 : std::string cmd("oks-copy.sh");
4726 :
4727 0 : if (get_verbose_mode())
4728 0 : cmd.append(" -v");
4729 :
4730 0 : cmd.append(" -s ");
4731 0 : cmd.append(source);
4732 :
4733 0 : cmd.append(" -d ");
4734 0 : cmd.append(destination);
4735 :
4736 0 : cmd.append(" -c ");
4737 0 : cmd.append(get_repository_version());
4738 :
4739 0 : auto start_usage = std::chrono::steady_clock::now();
4740 :
4741 0 : if (!p_silence) {
4742 0 : std::lock_guard lock(p_parallel_out_mutex);
4743 0 : log_timestamp() << "[OKS copy] => " << cmd << std::endl;
4744 0 : }
4745 :
4746 0 : CommandOutput cmd_out("oks-copy", this, cmd);
4747 0 : cmd_out.check_command_status(system(cmd.c_str()));
4748 :
4749 0 : if (!p_silence) {
4750 0 : std::lock_guard lock(p_parallel_out_mutex);
4751 0 : std::cout << cmd_out.cat2str();
4752 0 : log_timestamp()
4753 0 : << "[OKS copy] => done in "
4754 0 : << std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - start_usage).count() /
4755 0 : 1000.
4756 0 : << " ms" << std::endl;
4757 0 : }
4758 0 : }
4759 :
4760 : void
4761 0 : OksKernel::update_repository(const std::string& param, const std::string& val, RepositoryUpdateType update_type)
4762 : {
4763 0 : std::unique_lock lock(p_kernel_mutex);
4764 :
4765 0 : if (OksKernel::get_repository_root().empty())
4766 0 : throw RepositoryOperationFailed("update",
4767 0 : "the repository-root is not set (check environment variable TDAQ_DB_REPOSITORY)");
4768 :
4769 0 : if (get_user_repository_root().empty())
4770 0 : throw RepositoryOperationFailed(
4771 0 : "update", "the user-repository-root is not set (check environment variable TDAQ_DB_USER_REPOSITORY)");
4772 :
4773 0 : std::string cmd("oks-update.sh");
4774 :
4775 0 : if (get_verbose_mode())
4776 0 : cmd.append(" -v");
4777 :
4778 0 : cmd.append(" -u ");
4779 0 : cmd.append(get_user_repository_root());
4780 :
4781 0 : if (update_type == OksKernel::DiscardChanges)
4782 0 : cmd.append(" --force");
4783 0 : else if (update_type == OksKernel::MergeChanges)
4784 0 : cmd.append(" --merge");
4785 :
4786 0 : if (!param.empty()) {
4787 0 : cmd.append(" --");
4788 0 : cmd.append(param);
4789 0 : cmd.push_back(' ');
4790 0 : cmd.push_back('\"');
4791 0 : cmd.append(val);
4792 0 : cmd.push_back('\"');
4793 : }
4794 :
4795 0 : auto start_usage = std::chrono::steady_clock::now();
4796 :
4797 0 : if (!p_silence) {
4798 0 : std::lock_guard lock(p_parallel_out_mutex);
4799 0 : log_timestamp() << "[OKS update] => " << cmd << std::endl;
4800 0 : }
4801 :
4802 0 : CommandOutput cmd_out("oks-update", this, cmd);
4803 0 : cmd_out.check_command_status(system(cmd.c_str()));
4804 :
4805 0 : p_repository_update_ts = std::time(0);
4806 :
4807 0 : static std::string version_prefix("update oks version ");
4808 0 : std::string version = cmd_out.last_str();
4809 0 : std::string::size_type pos = version.find(version_prefix);
4810 :
4811 0 : if (pos == 0)
4812 0 : p_repository_version = version.substr(version_prefix.size());
4813 : else
4814 0 : throw RepositoryOperationFailed("commit", "cannot read oks version");
4815 :
4816 0 : if (!p_silence) {
4817 0 : std::lock_guard lock(p_parallel_out_mutex);
4818 0 : std::cout << cmd_out.cat2str();
4819 0 : log_timestamp()
4820 0 : << "[OKS update] => done in "
4821 0 : << std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - start_usage).count() /
4822 0 : 1000.
4823 0 : << " ms" << std::endl;
4824 0 : }
4825 0 : }
4826 :
4827 : void
4828 0 : OksKernel::commit_repository(const std::string& comments, const std::string& credentials)
4829 : {
4830 0 : std::unique_lock lock(p_kernel_mutex);
4831 :
4832 0 : if (OksKernel::get_repository_root().empty())
4833 0 : throw RepositoryOperationFailed("commit",
4834 0 : "the repository-root is not set (check environment variable TDAQ_DB_REPOSITORY)");
4835 :
4836 0 : if (get_user_repository_root().empty())
4837 0 : throw RepositoryOperationFailed(
4838 0 : "commit", "the user-repository-root is not set (check environment variable TDAQ_DB_USER_REPOSITORY)");
4839 :
4840 0 : std::string cmd("oks-commit.sh");
4841 :
4842 0 : if (get_verbose_mode())
4843 0 : cmd.append(" -v");
4844 :
4845 0 : cmd.append(" -u ");
4846 0 : cmd.append(get_user_repository_root());
4847 :
4848 0 : if (comments.empty() || std::all_of(comments.begin(), comments.end(), [](char c) { return std::isspace(c); }))
4849 0 : throw RepositoryOperationFailed("commit", "the commit message may not be empty");
4850 :
4851 0 : const std::string log_file_name(get_temporary_dir() + '/' + "oks-commit-msg." + std::to_string(getpid()) + ':' +
4852 0 : std::to_string(reinterpret_cast<std::uintptr_t>(this)) + ".txt");
4853 :
4854 0 : try {
4855 0 : std::ofstream f(log_file_name);
4856 :
4857 0 : f.exceptions(std::ostream::failbit | std::ostream::badbit);
4858 :
4859 0 : if (!f)
4860 0 : throw std::runtime_error("create message file failed");
4861 :
4862 0 : f << comments;
4863 0 : f.close();
4864 0 : } catch (const std::exception& ex) {
4865 0 : std::ostringstream ss;
4866 0 : ss << "cannot write to file \'" << log_file_name << "\' to save commit message: " << ex.what();
4867 0 : throw RepositoryOperationFailed("commit", ss.str());
4868 0 : }
4869 :
4870 0 : cmd.append(" -f \'");
4871 0 : cmd.append(log_file_name);
4872 0 : cmd.push_back('\'');
4873 :
4874 : // if (!credentials.empty())
4875 : // {
4876 : // try
4877 : // {
4878 : // auto token = daq::tokens::verify(credentials);
4879 :
4880 : // const std::string author = token.get_subject();
4881 :
4882 : // std::string email;
4883 :
4884 : // if (token.has_payload_claim("email"))
4885 : // email = token.get_payload_claim("email").as_string();
4886 :
4887 : // if (email.empty())
4888 : // email = author + '@' + get_domain_name();
4889 :
4890 : // OksSystem::User user(author);
4891 :
4892 : // cmd.append(" -n \'");
4893 : // cmd.append(user.real_name());
4894 : // cmd.append("\' -e \'");
4895 : // cmd.append(email);
4896 : // cmd.push_back('\'');
4897 :
4898 : // static std::once_flag flag;
4899 :
4900 : // std::call_once(flag, []()
4901 : // {
4902 : // const char * opt = " -o SendEnv=OKS_COMMIT_USER";
4903 :
4904 : // std::string val;
4905 :
4906 : // if (const char * s = getenv("GIT_SSH_COMMAND"))
4907 : // {
4908 : // if (strstr(s, opt) == nullptr)
4909 : // val = s;
4910 : // }
4911 : // else
4912 : // {
4913 : // val = "ssh";
4914 : // }
4915 :
4916 : // if (!val.empty())
4917 : // {
4918 : // val.append(opt);
4919 : // setenv("GIT_SSH_COMMAND", val.c_str(), 1);
4920 : // }
4921 : // }
4922 : // );
4923 :
4924 : // setenv("OKS_COMMIT_USER", credentials.c_str(), 1);
4925 : // }
4926 : // catch(const ers::Issue& ex)
4927 : // {
4928 : // std::ostringstream text;
4929 : // text << '\t' << ex;
4930 : // throw RepositoryOperationFailed("commit", text.str());
4931 : // }
4932 : // }
4933 :
4934 0 : auto start_usage = std::chrono::steady_clock::now();
4935 :
4936 0 : if (!p_silence) {
4937 0 : std::lock_guard lock(p_parallel_out_mutex);
4938 0 : log_timestamp() << "[OKS commit] => " << cmd << std::endl;
4939 0 : }
4940 :
4941 0 : CommandOutput cmd_out("oks-commit", this, cmd);
4942 :
4943 0 : int status = system(cmd.c_str());
4944 :
4945 0 : unlink(log_file_name.c_str());
4946 :
4947 0 : if (!credentials.empty())
4948 0 : unsetenv("OKS_COMMIT_USER");
4949 :
4950 0 : cmd_out.check_command_status(status);
4951 :
4952 0 : p_repository_update_ts = std::time(0);
4953 :
4954 0 : if (!p_silence) {
4955 0 : std::lock_guard lock(p_parallel_out_mutex);
4956 0 : std::cout << cmd_out.cat2str();
4957 0 : log_timestamp()
4958 0 : << "[OKS commit] => done in "
4959 0 : << std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - start_usage).count() /
4960 0 : 1000.
4961 0 : << " ms" << std::endl;
4962 0 : }
4963 :
4964 0 : static std::string version_prefix("commit oks version ");
4965 0 : std::string version = cmd_out.last_str();
4966 0 : std::string::size_type pos = version.find(version_prefix);
4967 :
4968 0 : if (pos == 0)
4969 0 : p_repository_version = version.substr(version_prefix.size());
4970 : else
4971 0 : throw RepositoryOperationFailed("commit", "cannot read oks version");
4972 0 : }
4973 :
4974 : void
4975 0 : OksKernel::tag_repository(const std::string& tag)
4976 : {
4977 0 : std::unique_lock lock(p_kernel_mutex);
4978 :
4979 0 : if (OksKernel::get_repository_root().empty())
4980 0 : throw RepositoryOperationFailed("tag",
4981 0 : "the repository-root is not set (check environment variable TDAQ_DB_REPOSITORY)");
4982 :
4983 0 : if (get_user_repository_root().empty())
4984 0 : throw RepositoryOperationFailed(
4985 0 : "tag", "the user-repository-root is not set (check environment variable TDAQ_DB_USER_REPOSITORY)");
4986 :
4987 0 : std::string cmd("oks-tag.sh");
4988 :
4989 0 : if (get_verbose_mode())
4990 0 : cmd.append(" -v");
4991 :
4992 0 : cmd.append(" -u ");
4993 0 : cmd.append(get_user_repository_root());
4994 :
4995 0 : if (tag.empty() || std::all_of(tag.begin(), tag.end(), [](char c) { return std::isspace(c); }))
4996 0 : throw RepositoryOperationFailed("tag", "the tag may not be empty");
4997 :
4998 0 : cmd.append(" -t \'");
4999 0 : cmd.append(tag);
5000 0 : cmd.append("\' -c \'");
5001 0 : cmd.append(get_repository_version());
5002 0 : cmd.push_back('\'');
5003 :
5004 0 : auto start_usage = std::chrono::steady_clock::now();
5005 :
5006 0 : if (!p_silence) {
5007 0 : std::lock_guard lock(p_parallel_out_mutex);
5008 0 : log_timestamp() << "[OKS tag] => " << cmd << std::endl;
5009 0 : }
5010 :
5011 0 : CommandOutput cmd_out("oks-tag", this, cmd);
5012 0 : cmd_out.check_command_status(system(cmd.c_str()));
5013 :
5014 0 : if (!p_silence) {
5015 0 : std::lock_guard lock(p_parallel_out_mutex);
5016 0 : std::cout << cmd_out.cat2str();
5017 0 : log_timestamp()
5018 0 : << "[OKS tag] => done in "
5019 0 : << std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - start_usage).count() /
5020 0 : 1000.
5021 0 : << " ms" << std::endl;
5022 0 : }
5023 0 : }
5024 :
5025 : std::list<std::string>
5026 0 : OksKernel::get_repository_versions_diff(const std::string& sha1, const std::string& sha2)
5027 : {
5028 0 : std::unique_lock lock(p_kernel_mutex);
5029 :
5030 0 : if (OksKernel::get_repository_root().empty())
5031 0 : throw RepositoryOperationFailed("diff",
5032 0 : "the repository-root is not set (check environment variable TDAQ_DB_REPOSITORY)");
5033 :
5034 0 : if (get_user_repository_root().empty())
5035 0 : throw RepositoryOperationFailed(
5036 0 : "diff", "the user-repository-root is not set (check environment variable TDAQ_DB_USER_REPOSITORY)");
5037 :
5038 0 : std::string cmd("oks-diff.sh");
5039 :
5040 0 : if (get_verbose_mode())
5041 0 : cmd.append(" -v");
5042 :
5043 0 : cmd.append(" -u ");
5044 0 : cmd.append(get_user_repository_root());
5045 :
5046 0 : if (!sha1.empty() && !sha2.empty()) {
5047 0 : cmd.append(" --sha ");
5048 0 : cmd.append(sha1);
5049 0 : cmd.push_back(' ');
5050 0 : cmd.append(sha2);
5051 : } else {
5052 0 : cmd.append(" --unmerged");
5053 : }
5054 :
5055 0 : CommandOutput cmd_out("oks-diff", this, cmd);
5056 0 : cmd_out.check_command_status(system(cmd.c_str()));
5057 :
5058 0 : std::string output = cmd_out.cat2str();
5059 :
5060 0 : if (!p_silence) {
5061 0 : std::lock_guard lock(p_parallel_out_mutex);
5062 0 : std::cout << output << "[OKS get_diff] => done" << std::endl;
5063 0 : }
5064 :
5065 0 : std::istringstream s(output);
5066 :
5067 0 : std::string line;
5068 0 : bool found = false;
5069 :
5070 0 : const char* diff_pattern = "git diff --name-only ";
5071 :
5072 0 : while (std::getline(s, line))
5073 0 : if (line.find(diff_pattern) == 0) {
5074 : found = true;
5075 : break;
5076 : }
5077 :
5078 0 : if (found == false) {
5079 0 : std::ostringstream text;
5080 0 : text << "cannot find \"" << diff_pattern << "\" pattern in output of oks-diff.sh:\n" << output;
5081 0 : throw RepositoryOperationFailed("get_diff", text.str());
5082 0 : }
5083 :
5084 0 : std::list<std::string> result;
5085 :
5086 0 : while (std::getline(s, line))
5087 0 : result.push_back(line);
5088 :
5089 0 : return result;
5090 0 : }
5091 :
5092 : static bool
5093 0 : check_relevant(const std::set<std::string>& loaded_files, const OksRepositoryVersion& ver)
5094 : {
5095 0 : for (const auto& x : ver.m_files)
5096 0 : if (loaded_files.find(x) != loaded_files.end())
5097 0 : return true;
5098 :
5099 0 : return false;
5100 : }
5101 :
5102 : std::vector<OksRepositoryVersion>
5103 0 : OksKernel::get_repository_versions(bool skip_irrelevant, const std::string& command_line)
5104 : {
5105 0 : std::string cmd("oks-log.sh");
5106 :
5107 0 : if (get_verbose_mode())
5108 0 : cmd.append(" -v");
5109 :
5110 0 : cmd.append(" -u ");
5111 0 : cmd.append(get_user_repository_root());
5112 :
5113 0 : cmd.append(command_line);
5114 :
5115 0 : CommandOutput cmd_out("oks-log", this, cmd);
5116 0 : cmd_out.check_command_status(system(cmd.c_str()));
5117 :
5118 0 : std::string output = cmd_out.cat2str();
5119 :
5120 0 : std::istringstream s(output);
5121 :
5122 0 : std::string line;
5123 0 : bool found = false;
5124 :
5125 0 : while (std::getline(s, line))
5126 0 : if (line.find("git log ") == 0) {
5127 : found = true;
5128 : break;
5129 : }
5130 :
5131 0 : if (found == false) {
5132 0 : std::ostringstream text;
5133 0 : text << "cannot find \"git log\" pattern in output of oks-log.sh:\n" << output;
5134 0 : throw RepositoryOperationFailed("log", text.str());
5135 0 : }
5136 :
5137 0 : std::set<std::string> loaded_files;
5138 :
5139 0 : if (skip_irrelevant) {
5140 0 : std::size_t len = get_user_repository_root().size() + 1;
5141 :
5142 0 : for (const auto& x : p_schema_files)
5143 0 : loaded_files.emplace(x.first->substr(len));
5144 :
5145 0 : for (const auto& x : p_data_files)
5146 0 : loaded_files.emplace(x.first->substr(len));
5147 : }
5148 :
5149 0 : std::vector<OksRepositoryVersion> vec;
5150 0 : OksRepositoryVersion ver;
5151 :
5152 0 : found = false;
5153 :
5154 0 : while (std::getline(s, line)) {
5155 0 : if (found == false) {
5156 0 : ver.clear();
5157 :
5158 0 : std::string::size_type idx1 = 0;
5159 0 : std::string::size_type idx2 = line.find('|');
5160 :
5161 0 : if (idx2 != std::string::npos) {
5162 0 : ver.m_commit_hash = line.substr(idx1, idx2);
5163 0 : idx1 = idx2 + 1;
5164 0 : idx2 = line.find('|', idx1);
5165 :
5166 0 : if (idx2 != std::string::npos) {
5167 0 : ver.m_user = line.substr(idx1, idx2 - idx1);
5168 0 : idx1 = idx2 + 1;
5169 0 : idx2 = line.find('|', idx1);
5170 :
5171 0 : if (idx2 != std::string::npos) {
5172 0 : ver.m_date = std::stol(line.substr(idx1, idx2 - idx1));
5173 0 : ver.m_comment = line.substr(idx2 + 1);
5174 : }
5175 : }
5176 : }
5177 :
5178 0 : if (!ver.m_comment.empty())
5179 : found = true;
5180 : else {
5181 0 : std::ostringstream text;
5182 0 : text << "unexpected line \"" << line << "\" in output of oks-log.sh:\n" << output;
5183 0 : throw RepositoryOperationFailed("log", text.str());
5184 0 : }
5185 : } else {
5186 0 : if (line.empty()) {
5187 0 : found = false;
5188 :
5189 0 : if (skip_irrelevant && check_relevant(loaded_files, ver) == false)
5190 0 : continue;
5191 :
5192 0 : vec.emplace_back(ver);
5193 : } else {
5194 0 : ver.m_files.push_back(line);
5195 : }
5196 : }
5197 : }
5198 :
5199 0 : if (found) {
5200 0 : if (skip_irrelevant == false || check_relevant(loaded_files, ver))
5201 0 : vec.emplace_back(ver);
5202 : }
5203 :
5204 0 : return vec;
5205 0 : }
5206 :
5207 : std::vector<OksRepositoryVersion>
5208 0 : OksKernel::get_repository_versions_by_hash(bool skip_irrelevant, const std::string& since, const std::string& until)
5209 : {
5210 0 : std::string command_line;
5211 :
5212 0 : if (!since.empty() && !until.empty()) {
5213 0 : command_line.push_back(' ');
5214 0 : command_line.append(since);
5215 0 : command_line.append("..");
5216 0 : command_line.append(until);
5217 0 : } else if (!since.empty()) {
5218 0 : command_line.push_back(' ');
5219 0 : command_line.append(since);
5220 0 : command_line.append("..origin/master");
5221 0 : } else if (!until.empty()) {
5222 0 : command_line.append(" ..");
5223 0 : command_line.append(until);
5224 : }
5225 :
5226 0 : return get_repository_versions(skip_irrelevant, command_line);
5227 0 : }
5228 :
5229 : static std::string
5230 0 : replace_datetime_spaces(const std::string& in)
5231 : {
5232 0 : std::string out(in);
5233 0 : std::replace(out.begin(), out.end(), ' ', 'T');
5234 0 : return out;
5235 : }
5236 :
5237 : std::vector<OksRepositoryVersion>
5238 0 : OksKernel::get_repository_versions_by_date(bool skip_irrelevant, const std::string& since, const std::string& until)
5239 : {
5240 0 : std::string command_line;
5241 :
5242 0 : if (!since.empty()) {
5243 0 : command_line.append(" --since ");
5244 0 : command_line.append(replace_datetime_spaces(since));
5245 : }
5246 :
5247 0 : if (!until.empty()) {
5248 0 : command_line.append(" --until ");
5249 0 : command_line.append(replace_datetime_spaces(until));
5250 : }
5251 :
5252 0 : command_line.append(" origin/master");
5253 :
5254 0 : return get_repository_versions(skip_irrelevant, command_line);
5255 0 : }
5256 :
5257 : std::string
5258 0 : OksKernel::read_repository_version()
5259 : {
5260 0 : std::unique_lock lock(p_kernel_mutex);
5261 :
5262 0 : if (OksKernel::get_repository_root().empty())
5263 0 : throw RepositoryOperationFailed("status",
5264 0 : "the repository-root is not set (check environment variable TDAQ_DB_REPOSITORY)");
5265 :
5266 0 : if (get_user_repository_root().empty())
5267 0 : throw RepositoryOperationFailed(
5268 0 : "status", "the user-repository-root is not set (check environment variable TDAQ_DB_USER_REPOSITORY)");
5269 :
5270 0 : std::string cmd("oks-version.sh");
5271 :
5272 0 : cmd.append(" -u ");
5273 0 : cmd.append(get_user_repository_root());
5274 :
5275 0 : CommandOutput cmd_out("oks-status", this, cmd);
5276 0 : cmd_out.check_command_status(system(cmd.c_str()));
5277 :
5278 0 : std::string output = cmd_out.cat2str();
5279 :
5280 0 : static std::string version_prefix("oks version ");
5281 0 : std::string version = cmd_out.last_str();
5282 0 : std::string::size_type pos = version.find(version_prefix);
5283 :
5284 0 : if (pos == 0)
5285 0 : p_repository_version = version.substr(version_prefix.size());
5286 : else {
5287 0 : std::ostringstream text;
5288 0 : text << "cannot read oks version from oks-version.sh output: \"" << output << '\"';
5289 0 : throw RepositoryOperationFailed("get version", text.str().c_str());
5290 0 : }
5291 :
5292 0 : return p_repository_version;
5293 0 : }
5294 :
5295 : void
5296 0 : OksKernel::get_updated_repository_files(std::set<std::string>& updated,
5297 : std::set<std::string>& added,
5298 : std::set<std::string>& removed)
5299 : {
5300 0 : std::unique_lock lock(p_kernel_mutex);
5301 :
5302 0 : if (OksKernel::get_repository_root().empty())
5303 0 : throw RepositoryOperationFailed("status",
5304 0 : "the repository-root is not set (check environment variable TDAQ_DB_REPOSITORY)");
5305 :
5306 0 : if (get_user_repository_root().empty())
5307 0 : throw RepositoryOperationFailed(
5308 0 : "status", "the user-repository-root is not set (check environment variable TDAQ_DB_USER_REPOSITORY)");
5309 :
5310 0 : std::string cmd("oks-status.sh");
5311 :
5312 0 : cmd.append(" -u ");
5313 0 : cmd.append(get_user_repository_root());
5314 :
5315 0 : CommandOutput cmd_out("oks-status", this, cmd);
5316 0 : cmd_out.check_command_status(system(cmd.c_str()));
5317 :
5318 0 : std::string output = cmd_out.cat2str();
5319 :
5320 0 : if (p_verbose) {
5321 0 : std::lock_guard lock(p_parallel_out_mutex);
5322 0 : std::cout << output << "[OKS get_status] => done" << std::endl;
5323 0 : }
5324 :
5325 0 : std::istringstream s(output);
5326 :
5327 0 : std::string line;
5328 0 : bool found = false;
5329 :
5330 0 : const char* diff_pattern = "git status --porcelain";
5331 :
5332 0 : while (std::getline(s, line))
5333 0 : if (line.find(diff_pattern) == 0) {
5334 : found = true;
5335 : break;
5336 : }
5337 :
5338 0 : if (found == false) {
5339 0 : std::ostringstream text;
5340 0 : text << "cannot find \"" << diff_pattern << "\" pattern in output of oks-status.sh:\n" << output;
5341 0 : throw RepositoryOperationFailed("status", text.str());
5342 0 : }
5343 :
5344 0 : std::list<std::string> result;
5345 :
5346 0 : while (std::getline(s, line)) {
5347 0 : if (line.find(" D ") == 0)
5348 0 : removed.insert(line.substr(3));
5349 0 : else if (line.find(" M ") == 0)
5350 0 : updated.insert(line.substr(3));
5351 0 : else if (line.find("?? ") == 0)
5352 0 : added.insert(line.substr(3));
5353 : else {
5354 0 : std::ostringstream text;
5355 0 : text << "unexpected output \"" << line << "\" in output of oks-status.sh:\n" << output;
5356 0 : throw RepositoryOperationFailed("status", text.str());
5357 0 : }
5358 : }
5359 0 : }
5360 :
5361 : // /* PAM authentication */
5362 :
5363 : // struct creds_pam_t {
5364 : // const char * user;
5365 : // const char * pwd;
5366 : // };
5367 :
5368 : // static int
5369 : // get_credentials_pam (int num_msg, const struct pam_message **msg, struct pam_response **resp, void *appdata_ptr)
5370 : // {
5371 : // struct creds_pam_t * creds = (struct creds_pam_t *)appdata_ptr;
5372 : // struct pam_response * buf = (struct pam_response *)(malloc(num_msg * sizeof (struct pam_response))); /* released
5373 : // by PAM */
5374 :
5375 : // for(int k = 0; k < num_msg; ++ k) {
5376 : // const size_t STRING_SIZE = 1024;
5377 : // char * out;
5378 : // int error = 0;
5379 :
5380 : // buf[k].resp = (char*)malloc(STRING_SIZE);
5381 : // out = buf[k].resp;
5382 : // out[0] = 0;
5383 :
5384 : // if(msg[k]->msg) {
5385 : // if(!strcmp(msg[k]->msg, "login:") || !strcmp(msg[k]->msg, "login: ")) {
5386 : // TLOG_DEBUG( 1 ) << "process \'login\' PAM credential of user " << creds->user ;
5387 : // strncpy(out, creds->user, STRING_SIZE);
5388 : // }
5389 : // else if(!strcmp(msg[k]->msg, "Password: ") || !strcmp(msg[k]->msg, "LDAP Password: ")) {
5390 : // TLOG_DEBUG( 1 ) << "process \'password\' PAM credential of user " << creds->user ;
5391 : // strncpy (out, creds->pwd, STRING_SIZE);
5392 : // }
5393 : // else {
5394 : // error = 1;
5395 : // }
5396 : // }
5397 :
5398 : // if(error != 0) {
5399 : // std::cerr << "ERROR: unexpected request in process PAM credential: " << msg[k]->msg << std::endl;
5400 : // for (int j = 0; j <= k; ++ j) {
5401 : // free(buf[k].resp);
5402 : // }
5403 : // free(buf);
5404 : // return PAM_CONV_ERR;
5405 : // }
5406 :
5407 : // out[STRING_SIZE - 1] = 0;
5408 : // buf[k].resp_retcode = 0;
5409 : // }
5410 :
5411 : // *resp = buf;
5412 :
5413 : // return PAM_SUCCESS;
5414 : // }
5415 :
5416 : // void
5417 : // OksKernel::validate_credentials(const char * user, const char * passwd)
5418 : // {
5419 : // if(!user || !*user) {
5420 : // throw oks::AuthenticationFailure("user name is not given");
5421 : // }
5422 :
5423 : // if(!passwd || !*passwd) {
5424 : // throw oks::AuthenticationFailure("user password is not given");
5425 : // }
5426 :
5427 : // struct creds_pam_t creds = {user, passwd};
5428 : // struct pam_conv conv = {get_credentials_pam, &creds};
5429 :
5430 : // const char *service = "system-auth";
5431 : // pam_handle_t * pamh = 0;
5432 :
5433 : // int retval = pam_start(service, 0, &conv, &pamh);
5434 : // if (retval != PAM_SUCCESS) {
5435 : // std::ostringstream text;
5436 : // text << "pam_start() failed for service \'" << service << "\': " << pam_strerror (pamh, retval);
5437 : // throw oks::AuthenticationFailure(text.str());
5438 : // }
5439 :
5440 : // retval = pam_authenticate (pamh, 0);
5441 : // if (retval == PAM_SUCCESS) {
5442 : // TLOG_DEBUG( 1 ) << "user " << user << " was authenticated" ;
5443 : // }
5444 : // else {
5445 : // std::ostringstream text;
5446 : // text << "failed to authenticate user " << user << ": " << pam_strerror (pamh, retval);
5447 : // throw oks::AuthenticationFailure(text.str());
5448 : // }
5449 :
5450 : // retval = pam_end (pamh,retval);
5451 : // if (retval != PAM_SUCCESS) {
5452 : // std::ostringstream text;
5453 : // text << "pam_end() failed: " << pam_strerror (pamh, retval);
5454 : // throw oks::AuthenticationFailure(text.str());
5455 : // }
5456 : // }
5457 :
5458 : void
5459 187 : OksKernel::k_check_bind_classes_status() const noexcept
5460 : {
5461 187 : std::ostringstream out;
5462 :
5463 14226 : for (auto& c : p_classes)
5464 14039 : c.second->check_relationships(out, true);
5465 :
5466 187 : p_bind_classes_status = out.str();
5467 187 : }
5468 :
5469 : std::ostream&
5470 0 : log_timestamp(__LogSeverity__ severity)
5471 : {
5472 0 : auto now(std::chrono::system_clock::now());
5473 0 : auto time_since_epoch(now.time_since_epoch());
5474 0 : auto seconds_since_epoch(std::chrono::duration_cast<std::chrono::seconds>(time_since_epoch));
5475 :
5476 0 : std::time_t now_t(std::chrono::system_clock::to_time_t(std::chrono::system_clock::time_point(seconds_since_epoch)));
5477 :
5478 0 : char buff[128];
5479 0 : std::size_t len = std::strftime(buff, 128 - 16, "%Y-%b-%d %H:%M:%S.", std::localtime(&now_t));
5480 0 : sprintf(buff + len,
5481 : "%03d ",
5482 0 : (int)(std::chrono::duration_cast<std::chrono::milliseconds>(time_since_epoch).count() -
5483 0 : seconds_since_epoch.count() * 1000));
5484 :
5485 0 : std::ostream& s(severity <= Warning ? std::cerr : std::cout);
5486 :
5487 0 : s << buff << (severity == Error ? "ERROR " : severity == Warning ? "WARNING " : severity == Debug ? "DEBUG " : "");
5488 :
5489 0 : return s;
5490 : }
5491 :
5492 : } // namespace oks
5493 : } // namespace dunedaq
|
