Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
itminedu
agkyra
Commits
e7e1b4f4
Commit
e7e1b4f4
authored
May 06, 2015
by
Giorgos Korfiatis
Browse files
pop from candidates before excluding file
parent
18658a87
Changes
3
Hide whitespace changes
Inline
Side-by-side
agkyra/agkyra/syncer/localfs_client.py
View file @
e7e1b4f4
...
...
@@ -40,6 +40,10 @@ OS_NOT_A_DIR = 20
OS_NO_FILE_OR_DIR
=
2
exclude_regexes
=
[
"\.#"
,
"\.~"
,
"~\$"
,
"~.*\.tmp$"
,
"\..*\.swp$"
]
exclude_pattern
=
re
.
compile
(
'|'
.
join
(
exclude_regexes
))
class
DirMissing
(
BaseException
):
pass
...
...
@@ -462,8 +466,6 @@ class LocalfsFileClient(FileClient):
self
.
ROOTPATH
=
settings
.
local_root_path
self
.
CACHEPATH
=
settings
.
cache_path
self
.
get_db
=
settings
.
get_db
self
.
exclude_files_exp
=
re
.
compile
(
'.*\.tmp$'
)
self
.
exclude_dir_exp
=
re
.
compile
(
self
.
CACHEPATH
)
self
.
probe_candidates
=
common
.
LockedDict
()
def
list_candidate_files
(
self
,
forced
=
False
):
...
...
@@ -478,14 +480,9 @@ class LocalfsFileClient(FileClient):
for
dirpath
,
dirnames
,
files
in
os
.
walk
(
self
.
ROOTPATH
):
rel_dirpath
=
os
.
path
.
relpath
(
dirpath
,
start
=
self
.
ROOTPATH
)
logger
.
debug
(
"'%s' '%s'"
%
(
dirpath
,
rel_dirpath
))
# if self.exclude_dir_exp.match(dirpath):
# continue
if
rel_dirpath
!=
'.'
:
candidates
[
utils
.
to_standard_sep
(
rel_dirpath
)]
=
None
for
filename
in
files
:
# if self.exclude_files_exp.match(filename) or \
# self.exclude_dir_exp.match(filename):
# continue
if
rel_dirpath
==
'.'
:
prefix
=
""
else
:
...
...
@@ -502,13 +499,23 @@ class LocalfsFileClient(FileClient):
local_path
=
utils
.
join_path
(
self
.
ROOTPATH
,
name
)
return
local_path_changes
(
local_path
,
state
)
def
exclude_file
(
self
,
objname
):
parts
=
objname
.
split
(
common
.
OBJECT_DIRSEP
)
init_part
=
parts
[
0
]
if
init_part
in
[
self
.
settings
.
cache_name
]:
return
True
final_part
=
parts
[
-
1
]
return
exclude_pattern
.
match
(
final_part
)
def
start_probing_file
(
self
,
objname
,
old_state
,
ref_state
,
assumed_info
=
None
,
callback
=
None
):
if
old_state
.
serial
!=
ref_state
.
serial
:
logger
.
warning
(
"Serial mismatch in probing path '%s'"
%
objname
)
return
cached_info
=
self
.
probe_candidates
.
pop
(
objname
)
if
self
.
exclude_file
(
objname
):
logger
.
warning
(
"Ignoring probe archive: %s, object: %s"
%
(
old_state
.
archive
,
objname
))
return
live_info
=
(
self
.
_local_path_changes
(
objname
,
old_state
)
if
cached_info
is
None
else
cached_info
)
if
live_info
is
None
:
...
...
agkyra/agkyra/syncer/pithos_client.py
View file @
e7e1b4f4
...
...
@@ -323,12 +323,12 @@ class PithosFileClient(FileClient):
def
start_probing_file
(
self
,
objname
,
old_state
,
ref_state
,
assumed_info
=
None
,
callback
=
None
):
info
=
old_state
.
info
cached_info
=
self
.
probe_candidates
.
pop
(
objname
)
if
exclude_pattern
.
match
(
objname
):
logger
.
warning
(
"Ignoring probe archive: %s, object: '%s'"
%
(
old_state
.
archive
,
objname
))
return
info
=
old_state
.
info
cached_info
=
self
.
probe_candidates
.
pop
(
objname
)
if
cached_info
is
None
:
obj
=
self
.
get_object
(
objname
)
live_info
=
self
.
get_object_live_info
(
obj
)
...
...
agkyra/agkyra/syncer/syncer.py
View file @
e7e1b4f4
...
...
@@ -16,7 +16,6 @@
import
time
import
threading
import
logging
import
re
from
agkyra.syncer
import
common
from
agkyra.syncer.setup
import
SyncerSettings
...
...
@@ -27,9 +26,6 @@ from agkyra.syncer import messaging, utils
logger
=
logging
.
getLogger
(
__name__
)
exclude_regexes
=
[
"\.#"
,
"\.~"
,
"~\$"
,
"~.*\.tmp$"
,
"\..*\.swp$"
]
exclude_pattern
=
re
.
compile
(
'|'
.
join
(
exclude_regexes
))
class
FileSyncer
(
object
):
...
...
@@ -100,20 +96,8 @@ class FileSyncer(object):
def
get_next_message
(
self
,
block
=
False
):
return
self
.
messager
.
get
(
block
=
block
)
def
exclude_file
(
self
,
objname
):
parts
=
objname
.
split
(
common
.
OBJECT_DIRSEP
)
init_part
=
parts
[
0
]
if
init_part
in
[
self
.
settings
.
cache_name
]:
return
True
final_part
=
parts
[
-
1
]
return
exclude_pattern
.
match
(
final_part
)
@
transaction
()
def
probe_file
(
self
,
archive
,
objname
,
assumed_info
=
None
):
if
self
.
exclude_file
(
objname
):
logger
.
warning
(
"Ignoring probe archive: %s, object: %s"
%
(
archive
,
objname
))
return
logger
.
info
(
"Probing archive: %s, object: '%s'"
%
(
archive
,
objname
))
db
=
self
.
get_db
()
client
=
self
.
clients
[
archive
]
...
...
@@ -141,10 +125,6 @@ class FileSyncer(object):
archive
=
live_state
.
archive
objname
=
live_state
.
objname
serial
=
live_state
.
serial
if
self
.
exclude_file
(
objname
):
logger
.
warning
(
"Ignoring update archive: %s, object: %s"
%
(
archive
,
objname
))
return
msg
=
messaging
.
UpdateMessage
(
archive
=
archive
,
objname
=
objname
,
serial
=
serial
,
logger
=
logger
)
self
.
messager
.
put
(
msg
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment