# -*- coding: utf-8 -*-
#
# Copyright © 2012 - 2018 Michal Čihař <michal@cihar.com>
#
# This file is part of Weblate <https://weblate.org/>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
#

from __future__ import unicode_literals

from glob import glob
import os
import sys
import time
import fnmatch
import re

from django.conf import settings
from django.db import models, transaction
from django.utils.translation import ugettext as _, ugettext_lazy
from django.utils.encoding import python_2_unicode_compatible, force_text
from django.utils.functional import cached_property
from django.core.mail import mail_admins
from django.core.exceptions import ValidationError
from django.urls import reverse
from django.core.cache import cache
from django.utils import timezone

from weblate.formats import ParseError
from weblate.formats.models import FILE_FORMATS
from weblate.trans.mixins import URLMixin, PathMixin
from weblate.trans.fields import RegexField
from weblate.utils import messages
from weblate.utils.site import get_site_url
from weblate.utils.state import STATE_TRANSLATED, STATE_FUZZY
from weblate.utils.errors import report_error
from weblate.trans.util import (
    is_repo_link, cleanup_repo_url, cleanup_path, path_separator,
    PRIORITY_CHOICES,
)
from weblate.trans.signals import (
    vcs_post_push, vcs_post_update, translation_post_add
)
from weblate.vcs.base import RepositoryException
from weblate.vcs.models import VCS_REGISTRY
from weblate.utils.stats import ComponentStats
from weblate.trans.models.translation import Translation
from weblate.trans.validators import (
    validate_filemask, validate_autoaccept, validate_check_flags,
)
from weblate.lang.models import Language
from weblate.trans.models.change import Change
from weblate.utils.validators import validate_repoweb, validate_render


DEFAULT_COMMIT_MESSAGE = (
    'Translated using Weblate ({{ language_name }})\n\n'
    'Currently translated at {{ stats.translated_percent }}% '
    '({{ stats.translated }} of {{ stats.all }} strings)\n\n'
    'Translation: {{ project_name }}/{{ component_name }}\n'
    'Translate-URL: {{ url }}'
)

DEFAULT_ADD_MESSAGE = (
    'Added translation using Weblate ({{ language_name }})\n\n'
)

DEFAULT_DELETE_MESSAGE = (
    'Deleted translation using Weblate ({{ language_name }})\n\n'
)

NEW_LANG_CHOICES = (
    ('contact', ugettext_lazy('Use contact form')),
    ('url', ugettext_lazy('Point to translation instructions URL')),
    ('add', ugettext_lazy('Automatically add language file')),
    ('none', ugettext_lazy('No adding of language')),
)
MERGE_CHOICES = (
    ('merge', ugettext_lazy('Merge')),
    ('rebase', ugettext_lazy('Rebase')),
)


def perform_on_link(func):
    """Decorator to handle repository link"""
    def on_link_wrapper(self, *args, **kwargs):
        if self.is_repo_link:
            # Call same method on linked component
            return getattr(self.linked_component, func.__name__)(
                *args, **kwargs
            )
        return func(self, *args, **kwargs)
    return on_link_wrapper


class ComponentQuerySet(models.QuerySet):
    # pylint: disable=no-init

    def prefetch(self):
        return self.select_related(
            'project'
        )

    def get_linked(self, val):
        """Return component for linked repo."""
        if not is_repo_link(val):
            return None
        project, component = val[10:].split('/', 1)
        return self.get(slug=component, project__slug=project)


@python_2_unicode_compatible
class Component(models.Model, URLMixin, PathMixin):
    name = models.CharField(
        verbose_name=ugettext_lazy('Component name'),
        max_length=settings.COMPONENT_NAME_LENGTH,
        help_text=ugettext_lazy('Name to display')
    )
    slug = models.SlugField(
        verbose_name=ugettext_lazy('URL slug'),
        max_length=settings.COMPONENT_NAME_LENGTH,
        help_text=ugettext_lazy('Name used in URLs and file names.')
    )
    project = models.ForeignKey(
        'Project',
        verbose_name=ugettext_lazy('Project'),
        on_delete=models.deletion.CASCADE,
    )
    vcs = models.CharField(
        verbose_name=ugettext_lazy('Version control system'),
        max_length=20,
        help_text=ugettext_lazy(
            'Version control system to use to access your '
            'repository with translations.'
        ),
        choices=VCS_REGISTRY.get_choices(),
        default=settings.DEFAULT_VCS,
    )
    repo = models.CharField(
        verbose_name=ugettext_lazy('Source code repository'),
        max_length=200,
        help_text=ugettext_lazy(
            'URL of a repository, use weblate://project/component '
            'for sharing with other component.'
        ),
    )
    linked_component = models.ForeignKey(
        'Component',
        verbose_name=ugettext_lazy('Project'),
        on_delete=models.deletion.CASCADE,
        null=True,
        editable=False,
    )
    push = models.CharField(
        verbose_name=ugettext_lazy('Repository push URL'),
        max_length=200,
        help_text=ugettext_lazy(
            'URL of a push repository, pushing is disabled if empty.'
        ),
        blank=True
    )
    repoweb = models.URLField(
        verbose_name=ugettext_lazy('Repository browser'),
        help_text=ugettext_lazy(
            'Link to repository browser, use %(branch)s for branch, '
            '%(file)s and %(line)s as filename and line placeholders.'
        ),
        validators=[validate_repoweb],
        blank=True,
    )
    git_export = models.CharField(
        verbose_name=ugettext_lazy('Exported repository URL'),
        max_length=200,
        help_text=ugettext_lazy(
            'URL of a repository where users can fetch changes from Weblate'
        ),
        blank=True
    )
    report_source_bugs = models.EmailField(
        verbose_name=ugettext_lazy('Source string bug report address'),
        help_text=ugettext_lazy(
            'Email address where errors in source string will be reported, '
            'keep empty for no emails.'
        ),
        max_length=254,
        blank=True,
    )
    branch = models.CharField(
        verbose_name=ugettext_lazy('Repository branch'),
        max_length=50,
        help_text=ugettext_lazy('Repository branch to translate'),
        default='',
        blank=True
    )
    filemask = models.CharField(
        verbose_name=ugettext_lazy('File mask'),
        max_length=200,
        validators=[validate_filemask],
        help_text=ugettext_lazy(
            'Path of files to translate relative to repository root,'
            ' use * instead of language code, '
            'for example: po/*.po or locale/*/LC_MESSAGES/django.po.'
        )
    )
    template = models.CharField(
        verbose_name=ugettext_lazy('Monolingual base language file'),
        max_length=200,
        blank=True,
        help_text=ugettext_lazy(
            'Filename of translations base file, which contains all strings '
            'and their source; this is recommended to use '
            'for monolingual translation formats.'
        )
    )
    edit_template = models.BooleanField(
        verbose_name=ugettext_lazy('Edit base file'),
        default=True,
        help_text=ugettext_lazy(
            'Whether users will be able to edit base file '
            'for monolingual translations.'
        )
    )
    new_base = models.CharField(
        verbose_name=ugettext_lazy('Base file for new translations'),
        max_length=200,
        blank=True,
        help_text=ugettext_lazy(
            'Filename of file which is used for creating new translations. '
            'For Gettext choose .pot file.'
        )
    )
    file_format = models.CharField(
        verbose_name=ugettext_lazy('File format'),
        max_length=50,
        default='auto',
        choices=FILE_FORMATS.get_choices(),
        help_text=ugettext_lazy(
            'Automatic detection might fail for some formats '
            'and is slightly slower.'
        ),
    )

    locked = models.BooleanField(
        verbose_name=ugettext_lazy('Locked'),
        default=False,
        help_text=ugettext_lazy(
            'Whether component is locked for translation updates.'
        )
    )
    allow_translation_propagation = models.BooleanField(
        verbose_name=ugettext_lazy('Allow translation propagation'),
        default=settings.DEFAULT_TRANSLATION_PROPAGATION,
        db_index=True,
        help_text=ugettext_lazy(
            'Whether translation updates in other components '
            'will cause automatic translation in this one'
        )
    )
    save_history = models.BooleanField(
        verbose_name=ugettext_lazy('Save translation history'),
        default=True,
        help_text=ugettext_lazy(
            'Whether Weblate should keep history of translations'
        )
    )
    enable_suggestions = models.BooleanField(
        verbose_name=ugettext_lazy('Enable suggestions'),
        default=True,
        help_text=ugettext_lazy(
            'Whether to allow translation suggestions at all.'
        )
    )
    suggestion_voting = models.BooleanField(
        verbose_name=ugettext_lazy('Suggestion voting'),
        default=False,
        help_text=ugettext_lazy(
            'Whether users can vote for suggestions.'
        )
    )
    suggestion_autoaccept = models.PositiveSmallIntegerField(
        verbose_name=ugettext_lazy('Autoaccept suggestions'),
        default=0,
        help_text=ugettext_lazy(
            'Automatically accept suggestions with this number of votes,'
            ' use 0 to disable.'
        ),
        validators=[validate_autoaccept],
    )
    check_flags = models.TextField(
        verbose_name=ugettext_lazy('Quality checks flags'),
        default='',
        help_text=ugettext_lazy(
            'Additional comma-separated flags to influence quality checks, '
            'check documentation for possible values.'
        ),
        validators=[validate_check_flags],
        blank=True,
    )

    # Licensing
    license = models.CharField(
        verbose_name=ugettext_lazy('Translation license'),
        max_length=150,
        blank=True,
        default='',
        help_text=ugettext_lazy(
            'Optional short summary of license used for translations.'
        ),
    )
    license_url = models.URLField(
        verbose_name=ugettext_lazy('License URL'),
        blank=True,
        default='',
        help_text=ugettext_lazy('Optional URL with license details.'),
    )
    agreement = models.TextField(
        verbose_name=ugettext_lazy('Contributor agreement'),
        blank=True,
        default='',
        help_text=ugettext_lazy(
            'Agreement which needs to be approved before user can '
            'translate this component.'
        )
    )

    # Adding new language
    new_lang = models.CharField(
        verbose_name=ugettext_lazy('New translation'),
        max_length=10,
        choices=NEW_LANG_CHOICES,
        default='add',
        help_text=ugettext_lazy(
            'How to handle requests for creating new translations. '
            'Please note that availability of choices depends on '
            'the file format.'
        ),
    )

    # VCS config
    merge_style = models.CharField(
        verbose_name=ugettext_lazy('Merge style'),
        max_length=10,
        choices=MERGE_CHOICES,
        default='merge',
        help_text=ugettext_lazy(
            'Define whether Weblate should merge upstream repository '
            'or rebase changes onto it.'
        ),
    )
    commit_message = models.TextField(
        verbose_name=ugettext_lazy('Commit message when translating'),
        help_text=ugettext_lazy(
            'You can use template language for various information, '
            'please check documentation for more details.'
        ),
        validators=[validate_render],
        default=DEFAULT_COMMIT_MESSAGE,
    )
    add_message = models.TextField(
        verbose_name=ugettext_lazy('Commit message when adding translation'),
        help_text=ugettext_lazy(
            'You can use template language for various information, '
            'please check documentation for more details.'
        ),
        validators=[validate_render],
        default=DEFAULT_ADD_MESSAGE,
    )
    delete_message = models.TextField(
        verbose_name=ugettext_lazy('Commit message when removing translation'),
        help_text=ugettext_lazy(
            'You can use template language for various information, '
            'please check documentation for more details.'
        ),
        validators=[validate_render],
        default=DEFAULT_DELETE_MESSAGE,
    )
    committer_name = models.CharField(
        verbose_name=ugettext_lazy('Committer name'),
        max_length=200,
        default=settings.DEFAULT_COMMITER_NAME,
    )
    committer_email = models.EmailField(
        verbose_name=ugettext_lazy('Committer email'),
        max_length=254,
        default=settings.DEFAULT_COMMITER_EMAIL,
    )
    push_on_commit = models.BooleanField(
        verbose_name=ugettext_lazy('Push on commit'),
        default=settings.DEFAULT_PUSH_ON_COMMIT,
        help_text=ugettext_lazy(
            'Whether the repository should be pushed upstream on every commit.'
        ),
    )
    commit_pending_age = models.IntegerField(
        verbose_name=ugettext_lazy('Age of changes to commit'),
        default=settings.COMMIT_PENDING_HOURS,
        help_text=ugettext_lazy(
            'Time in hours after which any pending changes will be '
            'committed to the VCS.'
        ),
    )

    language_regex = RegexField(
        verbose_name=ugettext_lazy('Language filter'),
        max_length=200,
        default='^[^.]+$',
        help_text=ugettext_lazy(
            'Regular expression which is used to filter '
            'translation when scanning for file mask.'
        ),
    )

    priority = models.IntegerField(
        default=100,
        choices=PRIORITY_CHOICES,
        verbose_name=_('Priority'),
        help_text=_(
            'Components with higher priority are offered first to translators.'
        ),
    )

    objects = ComponentQuerySet.as_manager()

    is_lockable = True
    _reverse_url_name = 'component'

    class Meta(object):
        ordering = ['priority', 'project__name', 'name']
        unique_together = (
            ('project', 'name'),
            ('project', 'slug'),
        )
        app_label = 'trans'
        verbose_name = ugettext_lazy('Component')
        verbose_name_plural = ugettext_lazy('Components')

    def __init__(self, *args, **kwargs):
        """Constructor to initialize some cache properties."""
        super(Component, self).__init__(*args, **kwargs)
        self._file_format = None
        self.stats = ComponentStats(self)
        self.addons_cache = {}

    @property
    def filemask_re(self):
        return re.compile(
            fnmatch.translate(self.filemask).replace('.*', '(.*)')
        )

    @cached_property
    def log_prefix(self):
        return '/'.join((self.project.slug, self.slug))

    def get_reverse_url_kwargs(self):
        """Return kwargs for URL reversing."""
        return {
            'project': self.project.slug,
            'component': self.slug
        }

    def get_widgets_url(self):
        """Return absolute URL for widgets."""
        return get_site_url(
            '{0}?component={1}'.format(
                reverse('widgets', kwargs={'project': self.project.slug}),
                self.slug,
            )
        )

    def get_share_url(self):
        """Return absolute URL usable for sharing."""
        return get_site_url(
            reverse('engage', kwargs={'project': self.project.slug})
        )

    def __str__(self):
        return '/'.join((force_text(self.project), self.name))

    @perform_on_link
    def _get_path(self):
        """Return full path to component VCS repository."""
        return os.path.join(self.project.full_path, self.slug)

    @perform_on_link
    def can_push(self):
        """Return true if push is possible for this component."""
        return self.push != '' and self.push is not None

    @property
    def is_repo_link(self):
        """Check whether repository is just a link for other one."""
        return is_repo_link(self.repo)

    def can_add_language(self):
        """Return true if new languages can be added."""
        return self.new_lang != 'none'

    @cached_property
    def repository(self):
        """Get VCS repository object."""
        if self.is_repo_link:
            return self.linked_component.repository
        repository = VCS_REGISTRY[self.vcs](
            self.full_path, self.branch, self
        )
        cache_key = 'sp-config-check-{}'.format(self.pk)
        if cache.get(cache_key) is None:
            with repository.lock:
                repository.check_config()
            cache.set(cache_key, True, 86400)

        return repository

    def get_last_remote_commit(self):
        """Return latest remote commit we know."""
        return self.repository.get_revision_info(
            self.repository.last_remote_revision
        )

    @perform_on_link
    def get_repo_url(self):
        """Return link to repository."""
        if not settings.HIDE_REPO_CREDENTIALS:
            return self.repo
        return cleanup_repo_url(self.repo)

    @perform_on_link
    def get_repo_branch(self):
        """Return branch in repository."""
        return self.branch

    @perform_on_link
    def get_export_url(self):
        """Return URL of exported VCS repository."""
        return self.git_export

    def get_repoweb_link(self, filename, line):
        """Generate link to source code browser for given file and line

        For linked repositories, it is possible to override linked
        repository path here.
        """
        if not self.repoweb:
            if self.is_repo_link:
                return self.linked_component.get_repoweb_link(filename, line)
            return None

        return self.repoweb % {
            'file': filename,
            '../file': filename.split('/', 1)[-1],
            '../../file': filename.split('/', 2)[-1],
            '../../../file': filename.split('/', 3)[-1],
            'line': line,
            'branch': self.branch
        }

    @perform_on_link
    def update_remote_branch(self, validate=False):
        """Pull from remote repository."""
        # Update
        self.log_info('updating repository')
        try:
            with self.repository.lock:
                start = time.time()
                self.repository.update_remote()
                timediff = time.time() - start
                self.log_info('update took %.2f seconds', timediff)
                for line in self.repository.last_output.splitlines():
                    self.log_debug('update: %s', line)
            return True
        except RepositoryException as error:
            error_text = force_text(error)
            self.log_error('failed to update repository: %s', error_text)
            if validate:
                if 'Host key verification failed' in error_text:
                    raise ValidationError(_(
                        'Failed to verify SSH host key, please add '
                        'them in SSH page in the admin interface.'
                    ))
                raise ValidationError(
                    _('Failed to fetch repository: %s') % error_text
                )
            return False

    def configure_repo(self, validate=False):
        """Ensure repository is correctly configured"""
        if self.is_repo_link:
            return

        with self.repository.lock:
            self.repository.configure_remote(self.repo, self.push, self.branch)
            self.repository.set_committer(
                self.committer_name,
                self.committer_email
            )

            self.update_remote_branch(validate)

    def configure_branch(self):
        """Ensure local tracking branch exists and is checkouted."""
        if self.is_repo_link:
            return

        with self.repository.lock:
            self.repository.configure_branch(self.branch)

    @perform_on_link
    def do_update(self, request=None, method=None):
        """Wrapper for doing repository update"""
        # Hold lock all time here to avoid somebody writing between commit
        # and merge/rebase.
        with self.repository.lock:
            # pull remote
            if not self.update_remote_branch():
                return False

            # do we have something to merge?
            try:
                needs_merge = self.repo_needs_merge()
            except RepositoryException:
                # Not yet configured repository
                needs_merge = True

            if not needs_merge and method != 'rebase':
                return True

            # commit possible pending changes
            self.commit_pending(request, skip_push=True)

            # update local branch
            ret = self.update_branch(request, method=method)

        # create translation objects for all files
        try:
            self.create_translations(request=request)
        except ParseError:
            ret = False

        # Push after possible merge
        if ret:
            self.push_if_needed(request, do_update=False)

        return ret

    def push_if_needed(self, request, do_update=True, on_commit=True):
        """Wrapper to push if needed

        Checks for:

        * Enabled push on commit
        * Configured push
        * There is something to push
        """
        if on_commit and not self.push_on_commit:
            return True
        if not self.can_push():
            return True
        if not self.repo_needs_push():
            return True
        return self.do_push(
            request, force_commit=False, do_update=do_update
        )

    @perform_on_link
    def do_push(self, request, force_commit=True, do_update=True):
        """Wrapper for pushing changes to remote repo."""
        # Do we have push configured
        if not self.can_push():
            messages.error(
                request,
                _('Push is disabled for %s.') % force_text(self)
            )
            return False

        # Commit any pending changes
        if force_commit:
            self.commit_pending(request, skip_push=True)

        # Do we have anything to push?
        if not self.repo_needs_push():
            return True

        if do_update:
            # Update the repo
            self.do_update(request)

            # Were all changes merged?
            if self.repo_needs_merge():
                return False

        # Do actual push
        try:
            self.log_info('pushing to remote repo')
            with self.repository.lock:
                self.repository.push()

            Change.objects.create(
                action=Change.ACTION_PUSH, component=self,
                user=request.user if request else None,
            )

            vcs_post_push.send(sender=self.__class__, component=self)
            for component in self.get_linked_childs():
                vcs_post_push.send(
                    sender=component.__class__, component=component
                )

            return True
        except RepositoryException as error:
            self.log_error('failed to push on repo: %s', error)
            msg = 'Error:\n{0}'.format(str(error))
            mail_admins(
                'failed push on repo {0}'.format(force_text(self)),
                msg
            )
            Change.objects.create(
                action=Change.ACTION_FAILED_PUSH, component=self,
                target=force_text(error),
                user=request.user if request else None,
            )
            messages.error(
                request,
                _('Failed to push to remote branch on %s.') %
                force_text(self)
            )
            return False

    @perform_on_link
    def do_reset(self, request=None):
        """Wrapper for reseting repo to same sources as remote."""
        # First check we're up to date
        self.update_remote_branch()

        # Do actual reset
        try:
            self.log_info('reseting to remote repo')
            with self.repository.lock:
                self.repository.reset()

            Change.objects.create(
                action=Change.ACTION_RESET, component=self,
                user=request.user if request else None,
            )
        except RepositoryException as error:
            self.log_error('failed to reset on repo')
            msg = 'Error:\n{0}'.format(str(error))
            mail_admins(
                'failed reset on repo {0}'.format(force_text(self)),
                msg
            )
            messages.error(
                request,
                _('Failed to reset to remote branch on %s.') %
                force_text(self)
            )
            return False

        # create translation objects for all files
        self.create_translations(request=request)

        return True

    def get_repo_link_url(self):
        return 'weblate://{0}/{1}'.format(self.project.slug, self.slug)

    def get_linked_childs(self):
        """Return list of components which link repository to us."""
        return self.component_set.prefetch()

    def commit_pending(self, request, from_link=False, skip_push=False):
        """Check whether there is any translation which needs commit."""

        # If we're not recursing, call on parent
        if not from_link and self.is_repo_link:
            return self.linked_component.commit_pending(
                request, True, skip_push=skip_push
            )

        # Commit all translations
        for translation in self.translation_set.all():
            translation.commit_pending(request, skip_push=True)

        # Process linked projects
        for component in self.get_linked_childs():
            component.commit_pending(request, True, skip_push=True)

        if not from_link and not skip_push:
            self.push_if_needed(request)

        return True

    def handle_parse_error(self, error, translation=None):
        """Handler for parse error."""
        report_error(error, sys.exc_info())
        if translation is None:
            filename = self.template
        else:
            filename = translation.filename
        from weblate.accounts.notifications import notify_parse_error
        notify_parse_error(
            self,
            translation,
            str(error),
            filename
        )
        if self.id:
            Change.objects.create(
                component=self,
                action=Change.ACTION_PARSE_ERROR,
            )
        raise ParseError(str(error))

    @perform_on_link
    def update_branch(self, request=None, method=None):
        """Update current branch to match remote (if possible)."""
        if method is None:
            method = self.merge_style

        # Merge/rebase
        if method == 'rebase':
            method = self.repository.rebase
            error_msg = _('Failed to rebase our branch onto remote branch %s.')
            action = Change.ACTION_REBASE
            action_failed = Change.ACTION_FAILED_REBASE
        else:
            method = self.repository.merge
            error_msg = _('Failed to merge remote branch into %s.')
            action = Change.ACTION_MERGE
            action_failed = Change.ACTION_FAILED_MERGE

        with self.repository.lock:
            try:
                previous_head = self.repository.last_revision
                # Try to merge it
                method()
                self.log_info(
                    '%s remote into repo',
                    self.merge_style,
                )
                if self.id:
                    Change.objects.create(
                        component=self, action=action,
                        user=request.user if request else None,
                    )

                    # run post update hook
                    vcs_post_update.send(
                        sender=self.__class__,
                        component=self,
                        previous_head=previous_head
                    )
                    for component in self.get_linked_childs():
                        vcs_post_update.send(
                            sender=component.__class__,
                            component=component,
                            previous_head=previous_head
                        )
                return True
            except RepositoryException as error:
                # In case merge has failer recover
                error = error.get_message()
                status = self.repository.status()

                # Log error
                self.log_error(
                    'failed %s on repo: %s',
                    self.merge_style,
                    error
                )
                if self.id:
                    Change.objects.create(
                        component=self, action=action_failed, target=error,
                        user=request.user if request else None,
                    )

                # Notify subscribers and admins
                from weblate.accounts.notifications import notify_merge_failure
                notify_merge_failure(self, error, status)

                # Reset repo back
                method(abort=True)

                # Tell user (if there is any)
                messages.error(
                    request,
                    error_msg % force_text(self)
                )

                return False

    def get_mask_matches(self):
        """Return files matching current mask."""
        prefix = path_separator(os.path.join(self.full_path, ''))
        matches = set()
        for filename in glob(os.path.join(self.full_path, self.filemask)):
            path = path_separator(filename).replace(prefix, '')
            code = self.get_lang_code(path)
            if re.match(self.language_regex, code):
                matches.add(path)
            else:
                self.log_info('skipping language %s [%s]', code, path)

        # We want to list template among translations as well
        if self.has_template():
            if self.edit_template:
                matches.add(self.template)
            else:
                matches.discard(self.template)

        # Remove symlinked translations
        for filename in list(matches):
            resolved = self.repository.resolve_symlinks(filename)
            if resolved != filename and resolved in matches:
                matches.discard(filename)

        return sorted(matches)

    def create_translations(self, force=False, langs=None, request=None,
                            changed_template=False):
        """Load translations from VCS."""
        translations = set()
        languages = set()
        matches = self.get_mask_matches()
        for pos, path in enumerate(matches):
            with transaction.atomic():
                code = self.get_lang_code(path)
                if langs is not None and code not in langs:
                    self.log_info('skipping %s', path)
                    continue

                self.log_info(
                    'checking %s (%s) [%d/%d]',
                    path,
                    code,
                    pos + 1,
                    len(matches)
                )
                lang = Language.objects.auto_get_or_create(code=code)
                if lang.code in languages:
                    self.log_error('duplicate language found: %s', lang.code)
                    continue
                translation = Translation.objects.check_sync(
                    self, lang, code, path, force, request=request
                )
                translations.add(translation.id)
                languages.add(lang.code)
                # Remove fuzzy flag on template name change
                if changed_template:
                    translation.unit_set.filter(
                        state=STATE_FUZZY
                    ).update(
                        state=STATE_TRANSLATED
                    )

        # Delete possibly no longer existing translations
        if langs is None:
            todelete = self.translation_set.exclude(id__in=translations)
            if todelete.exists():
                with transaction.atomic():
                    self.log_info(
                        'removing stale translations: %s',
                        ','.join([trans.language.code for trans in todelete])
                    )
                    todelete.delete()

        # Process linked repos
        for component in self.get_linked_childs():
            self.log_info(
                'updating linked project %s',
                component
            )
            component.create_translations(force, langs, request=request)

        self.log_info('updating completed')

    def get_lang_code(self, path):
        """Parse language code from path."""
        # Parse filename
        matches = self.filemask_re.match(path)

        if not matches or not matches.lastindex:
            if path == self.template:
                return self.project.source_language.code
            return ''

        # Use longest matched code
        code = max(matches.groups(), key=len)

        # Remove possible encoding part
        if '.' in code and ('.utf' in code.lower() or '.iso' in code.lower()):
            return code.split('.')[0]
        return code

    def sync_git_repo(self, validate=False, skip_push=None):
        """Bring VCS repo in sync with current model."""
        if self.is_repo_link:
            return
        if skip_push is None:
            skip_push = validate
        self.configure_repo(validate)
        self.commit_pending(None, skip_push=skip_push)
        self.configure_branch()
        self.update_branch()

    def set_default_branch(self):
        """Set default VCS branch if empty"""
        if self.branch == '':
            self.branch = VCS_REGISTRY[self.vcs].default_branch

    def clean_repo_link(self):
        """Validate repository link."""
        try:
            repo = Component.objects.get_linked(self.repo)
            if repo is not None and repo.is_repo_link:
                raise ValidationError(
                    {'repo': _(
                        'Invalid link to a Weblate project, '
                        'can not link to linked repository!'
                    )}
                )
            if repo.pk == self.pk:
                raise ValidationError(
                    {'repo': _(
                        'Invalid link to a Weblate project, '
                        'can not link to self!'
                    )}
                )
        except (Component.DoesNotExist, ValueError):
            raise ValidationError(
                {'repo': _(
                    'Invalid link to a Weblate project, '
                    'use weblate://project/component.'
                )}
            )
        if self.push != '':
            raise ValidationError(
                {'push': _('Push URL is not used when repository is linked!')}
            )
        if self.git_export != '':
            raise ValidationError(
                {
                    'git_export':
                        _('Export URL is not used when repository is linked!')
                }
            )
        self.linked_component = Component.objects.get_linked(self.repo)

    def clean_lang_codes(self, matches):
        """Validate that there are no double language codes"""
        if not matches and not self.can_add_new_language():
            raise ValidationError(
                {'filemask': _('The mask did not match any files!')}
            )
        langs = set()
        translated_langs = set()
        for match in matches:
            code = self.get_lang_code(match)
            if not code:
                raise ValidationError({'filemask': _(
                    'Got empty language code for %s, please check filemask!'
                ) % match})
            lang = Language.objects.auto_get_or_create(code=code)
            if len(code) > 20:
                raise ValidationError({'filemask': _(
                    'Language code %s is too long, please check filemask!'
                ) % match})
            if code in langs:
                raise ValidationError(_(
                    'There are more files for single language (%s), please '
                    'adjust the mask and use components for translating '
                    'different resources.'
                ) % code)
            if lang.code in translated_langs:
                raise ValidationError(_(
                    'Multiple translations were mapped to a single language '
                    'code (%s). You should disable SIMPLIFY_LANGUAGES '
                    'to prevent Weblate mapping similar languages to one.'
                ) % lang.code)
            langs.add(code)
            translated_langs.add(lang.code)

    def clean_files(self, matches):
        """Validate whether we can parse translation files."""
        notrecognized = []
        errors = []
        dir_path = self.full_path
        for match in matches:
            try:
                parsed = self.file_format_cls.parse(
                    os.path.join(dir_path, match),
                )
                if not self.file_format_cls.is_valid(parsed.store):
                    errors.append('{0}: {1}'.format(
                        match, _('File does not seem to be valid!')
                    ))
            except ValueError:
                notrecognized.append(match)
            except Exception as error:
                errors.append('{0}: {1}'.format(match, str(error)))
        if notrecognized:
            msg = (
                _('Format of %d matched files could not be recognized.') %
                len(notrecognized)
            )
            raise ValidationError('{0}\n{1}'.format(
                msg,
                '\n'.join(notrecognized)
            ))
        if errors:
            raise ValidationError('{0}\n{1}'.format(
                (_('Failed to parse %d matched files!') % len(errors)),
                '\n'.join(errors)
            ))

    def clean_new_lang(self):
        """Validate new language choices."""
        if self.new_lang == 'add':
            filename = self.get_new_base_filename()
            if not self.file_format_cls.is_valid_base_for_new(filename):
                if filename is not None:
                    raise ValidationError(_(
                        'You have configured Weblate to add new translation '
                        'files, but did not provide base file to do that!'
                    ))
                raise ValidationError(_(
                    'Format of base file for new translations '
                    'was not recognized!'
                ))
        elif self.new_lang != 'add' and self.new_base:
            msg = _(
                'Base file for new translations is not used because of '
                'component settings. '
                'You probably want to enable automatic adding of new '
                'translations.'
            )
            raise ValidationError({'new_lang': msg, 'new_base': msg})

    def clean_template(self):
        """Validate template value."""
        # Test for unexpected template usage
        if self.template != '' and self.file_format_cls.monolingual is False:
            msg = _('You can not use base file with bilingual translation!')
            raise ValidationError({'template': msg, 'file_format': msg})

        # Special case for Gettext
        if self.template.endswith('.pot') and self.filemask.endswith('.po'):
            msg = _('Using .pot file as base file is not supported!')
            raise ValidationError({'template': msg})

        # Validate template loading
        if self.has_template():
            full_path = os.path.join(self.full_path, self.template)
            if not os.path.exists(full_path):
                msg = _('Template file not found!')
                raise ValidationError({'template': msg})

            try:
                self.template_store
            except ParseError as exc:
                msg = _('Failed to parse translation base file: %s') % str(exc)
                raise ValidationError({'template': msg})

            code = self.get_lang_code(self.template)
            if code:
                lang = Language.objects.auto_get_or_create(
                    code=code
                ).base_code()
                if lang != self.project.source_language.base_code():
                    msg = _(
                        'Template language ({0}) does not '
                        'match project source language ({1})!'
                    ).format(code, self.project.source_language.code)
                    raise ValidationError({'template': msg})

        elif self.file_format_cls.monolingual:
            msg = _(
                'You can not use monolingual translation without base file!'
            )
            raise ValidationError({'template': msg})

    def clean(self):
        """Validator fetches repository

        It tries to find translation files and it checks them for validity.
        """
        if self.new_lang == 'url' and self.project.instructions == '':
            msg = _(
                'Please either fill in instructions URL '
                'or use different option for adding new language.'
            )
            raise ValidationError({'new_lang': msg})

        if self.license == '' and self.license_url != '':
            msg = _(
                'License URL can not be used without license summary.'
            )
            raise ValidationError({'license_url': msg, 'license': msg})

        # Skip validation if we don't have valid project
        if self.project_id is None:
            return

        self.set_default_branch()

        # Check if we should rename
        if self.id:
            old = Component.objects.get(pk=self.id)
            self.check_rename(old)

            if old.vcs != self.vcs:
                # This could work, but the problem is that before changed
                # object is saved the linked repos still see old vcs leading
                # to horrible mess. Changing vcs from the manage.py shell
                # works fine though.
                msg = _('Changing version control system is not supported!')
                raise ValidationError({'vcs': msg})

        # Check file format
        if self.file_format not in FILE_FORMATS:
            msg = _('Unsupported file format: {0}').format(self.file_format)
            raise ValidationError({'file_format': msg})

        # Validate VCS repo
        try:
            self.sync_git_repo(True)
        except RepositoryException as exc:
            msg = _('Failed to update repository: %s') % exc
            raise ValidationError({'repo': msg})

        # Push repo is not used with link
        if self.is_repo_link:
            self.clean_repo_link()

        # Template validation
        self.clean_template()

        try:
            matches = self.get_mask_matches()

            # Verify language codes
            self.clean_lang_codes(matches)

            # Try parsing files
            self.clean_files(matches)
        except re.error:
            raise ValidationError(_(
                'Can not validate file matches due to invalid '
                'regular expression.'
            ))

        # New language options
        self.clean_new_lang()

        # Suggestions
        if (hasattr(self, 'suggestion_autoaccept') and
                self.suggestion_autoaccept and
                not self.suggestion_voting):
            msg = _(
                'Automatically accepting suggestions can work only with '
                'voting enabled!'
            )
            raise ValidationError(
                {'suggestion_autoaccept': msg, 'suggestion_voting': msg}
            )

    def get_template_filename(self):
        """Create absolute filename for template."""
        return os.path.join(self.full_path, self.template)

    def get_new_base_filename(self):
        """Create absolute filename for base file for new translations."""
        if not self.new_base:
            return None
        return os.path.join(self.full_path, self.new_base)

    def save(self, *args, **kwargs):
        """Save wrapper

        It updates backend repository and regenerates translation data.
        """
        self.set_default_branch()

        # Linked component cache
        self.linked_component = Component.objects.get_linked(self.repo)

        # Detect if VCS config has changed (so that we have to pull the repo)
        changed_git = True
        changed_setup = False
        changed_template = False
        changed_project = False
        if self.id:
            old = Component.objects.get(pk=self.id)
            changed_git = (
                (old.repo != self.repo) or
                (old.branch != self.branch) or
                (old.filemask != self.filemask) or
                (old.language_regex != self.language_regex)
            )
            changed_setup = (
                (old.file_format != self.file_format) or
                (old.edit_template != self.edit_template) or
                (old.template != self.template)
            )
            changed_template = (
                (old.edit_template != self.edit_template) and
                self.template
            )
            changed_project = (old.project_id != self.project_id)
            # Detect slug changes and rename git repo
            self.check_rename(old)
            # Rename linked repos
            if old.slug != self.slug:
                old.component_set.update(repo=self.get_repo_link_url())

        # Remove leading ./ from paths
        self.filemask = cleanup_path(self.filemask)
        self.template = cleanup_path(self.template)

        # Save/Create object
        super(Component, self).save(*args, **kwargs)

        # Configure git repo if there were changes
        if changed_git:
            self.sync_git_repo(skip_push=kwargs.get('force_insert', False))

        # Rescan for possibly new translations if there were changes, needs to
        # be done after actual creating the object above
        if changed_setup:
            self.create_translations(
                force=True,
                changed_template=changed_template
            )
        elif changed_git:
            self.create_translations()

        # Copy suggestions to new project
        if changed_project:
            old.project.suggestion_set.copy(self.project)

    def repo_needs_commit(self):
        """Check whether there are some not committed changes"""
        for translation in self.translation_set.all():
            if translation.unit_set.filter(pending=True).exists():
                return True
        return self.repository.needs_commit()

    def repo_needs_merge(self):
        """Check whether there is something to merge from remote repository"""
        return self.repository.needs_merge()

    def repo_needs_push(self):
        """Check whether there is something to push to remote repository"""
        return self.repository.needs_push()

    @property
    def file_format_name(self):
        return self.file_format_cls.name

    @property
    def file_format_cls(self):
        """Return file format object """
        if (self._file_format is None or
                self._file_format.name != self.file_format):
            self._file_format = FILE_FORMATS[self.file_format]
        return self._file_format

    def has_template(self):
        """Return true if component is using template for translation"""
        monolingual = self.file_format_cls.monolingual
        return (
            (monolingual or monolingual is None) and
            self.template and
            not self.template.endswith('.pot')
        )

    def load_template_store(self):
        """Load translate-toolkit store for template."""
        return self.file_format_cls.parse(
            self.get_template_filename(),
        )

    @cached_property
    def template_store(self):
        """Get translate-toolkit store for template."""
        # Do we need template?
        if not self.has_template():
            return None

        try:
            return self.load_template_store()
        except Exception as exc:
            self.handle_parse_error(exc)

    @property
    def last_change(self):
        """Return date of last change done in Weblate."""
        try:
            return Change.objects.content().filter(
                translation__component=self
            ).values_list(
                'timestamp', flat=True
            )[0]
        except IndexError:
            return None

    @cached_property
    def all_flags(self):
        """Return parsed list of flags."""
        return (
            self.check_flags.split(',') +
            list(self.file_format_cls.check_flags)
        )

    def can_add_new_language(self):
        """Wrapper to check if we can add new language."""
        if self.new_lang != 'add':
            return False

        base_filename = self.get_new_base_filename()
        if not self.file_format_cls.is_valid_base_for_new(base_filename):
            return False

        return True

    def add_new_language(self, language, request, send_signal=True):
        """Create new language file."""
        if not self.can_add_new_language():
            if request:
                messages.error(
                    request,
                    _('Failed to add new translation file!')
                )
            return False

        format_lang_code = self.file_format_cls.get_language_code(
            language.code
        )
        if re.match(self.language_regex, format_lang_code) is None:
            if request:
                messages.error(
                    request,
                    _('Given language is filtered by the language filter!')
                )
            return False

        base_filename = self.get_new_base_filename()

        filename = self.file_format_cls.get_language_filename(
            self.filemask,
            language.code
        )
        fullname = os.path.join(self.full_path, filename)

        # Ignore request if file exists (possibly race condition as
        # the processing of new language can take some time and user
        # can submit again)
        if os.path.exists(fullname):
            if request:
                Translation.objects.check_sync(
                    self, language, language.code, filename, request=request
                )
                messages.error(
                    request,
                    _('Translation file already exists!')
                )
            return False

        self.file_format_cls.add_language(
            fullname,
            language,
            base_filename
        )

        translation = Translation.objects.create(
            component=self,
            language=language,
            plural=language.plural,
            filename=filename,
            language_code=language.code,
            commit_message='__add__'
        )
        if send_signal:
            translation_post_add.send(
                sender=self.__class__,
                translation=translation
            )
        translation.git_commit(
            request,
            request.user.get_author_name()
            if request else 'Weblate <noreply@weblate.org>',
            timezone.now(),
            force_new=True,
        )
        translation.check_sync(
            force=True,
            request=request
        )
        return True

    def do_lock(self, user, lock=True):
        """Lock or unlock component."""
        self.locked = lock
        self.save(update_fields=['locked'])
        Change.objects.create(
            component=self,
            user=user,
            action=Change.ACTION_LOCK if lock else Change.ACTION_UNLOCK,
        )

    def get_editable_template(self):
        if not self.edit_template or not self.has_template():
            return None
        return self.translation_set.get(filename=self.template)

    def get_language_count(self):
        return self.translation_set.count()
