Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
itminedu
agkyra
Commits
ede6421e
Commit
ede6421e
authored
May 07, 2015
by
Giorgos Korfiatis
Browse files
probe all in single transaction BUT check probe_candidates on retry
parent
eebea533
Changes
3
Hide whitespace changes
Inline
Side-by-side
agkyra/agkyra/syncer/localfs_client.py
View file @
ede6421e
...
...
@@ -532,7 +532,7 @@ class LocalfsFileClient(FileClient):
final_part
=
parts
[
-
1
]
return
exclude_pattern
.
match
(
final_part
)
def
start_
prob
ing
_file
(
self
,
objname
,
old_state
,
ref_state
,
callback
=
None
):
def
prob
e
_file
(
self
,
objname
,
old_state
,
ref_state
):
with
self
.
probe_candidates
.
lock
()
as
d
:
cached_info
=
d
.
pop
(
objname
,
None
)
if
self
.
exclude_file
(
objname
):
...
...
@@ -545,8 +545,7 @@ class LocalfsFileClient(FileClient):
if
live_info
is
None
:
return
live_state
=
old_state
.
set
(
info
=
live_info
)
if
callback
is
not
None
:
callback
(
live_state
)
return
live_state
def
stage_file
(
self
,
source_state
):
return
LocalfsSourceHandle
(
self
.
settings
,
source_state
)
...
...
agkyra/agkyra/syncer/pithos_client.py
View file @
ede6421e
...
...
@@ -340,7 +340,7 @@ class PithosFileClient(FileClient):
PITHOS_TYPE
:
p_type
,
}
def
start_
prob
ing
_file
(
self
,
objname
,
old_state
,
ref_state
,
callback
=
None
):
def
prob
e
_file
(
self
,
objname
,
old_state
,
ref_state
):
info
=
old_state
.
info
with
self
.
probe_candidates
.
lock
()
as
d
:
cached_info
=
d
.
pop
(
objname
,
None
)
...
...
@@ -354,9 +354,8 @@ class PithosFileClient(FileClient):
else
:
live_info
=
cached_info
if
info
!=
live_info
:
if
callback
is
not
None
:
live_state
=
old_state
.
set
(
info
=
live_info
)
callback
(
live_state
)
live_state
=
old_state
.
set
(
info
=
live_info
)
return
live_state
def
stage_file
(
self
,
source_state
):
return
PithosSourceHandle
(
self
.
settings
,
source_state
)
...
...
agkyra/agkyra/syncer/syncer.py
View file @
ede6421e
...
...
@@ -97,6 +97,9 @@ class FileSyncer(object):
@
transaction
()
def
probe_file
(
self
,
archive
,
objname
):
return
self
.
_probe_file
(
archive
,
objname
)
def
_probe_file
(
self
,
archive
,
objname
):
logger
.
info
(
"Probing archive: %s, object: '%s'"
%
(
archive
,
objname
))
db
=
self
.
get_db
()
client
=
self
.
clients
[
archive
]
...
...
@@ -114,10 +117,10 @@ class FileSyncer(object):
logger
.
warning
(
"Serial mismatch in probing archive: %s, "
"object: '%s'"
%
(
archive
,
objname
))
return
client
.
start_probing_file
(
objname
,
db_state
,
ref_state
,
callback
=
self
.
update_file_state
)
live_state
=
client
.
probe_file
(
objname
,
db_state
,
ref_state
)
if
live_state
is
not
None
:
self
.
update_file_state
(
live_state
)
@
transaction
()
def
update_file_state
(
self
,
live_state
):
db
=
self
.
get_db
()
archive
=
live_state
.
archive
...
...
@@ -345,11 +348,12 @@ class FileSyncer(object):
return
set
(
db
.
list_deciding
(
archives
=
archives
,
sync
=
self
.
SYNC
))
@
transaction
()
def
probe_archive
(
self
,
archive
,
forced
=
False
):
client
=
self
.
clients
[
archive
]
candidates
=
client
.
list_candidate_files
(
forced
=
forced
)
for
objname
in
candidates
:
self
.
probe_file
(
archive
,
objname
)
self
.
_
probe_file
(
archive
,
objname
)
def
decide_archive
(
self
,
archive
):
for
objname
in
self
.
list_deciding
([
archive
]):
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment